From 58eac6dd03c087b24c772bbb21827223670c6824 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Thu, 6 Nov 2025 17:54:25 -0500 Subject: [PATCH 01/37] Export images to rootfs with docker --- README.md | 6 +- cmd/api/api/api_test.go | 1 - go.mod | 37 ++++++- go.sum | 108 +++++++++++++++++- lib/images/docker.go | 114 +++++++++++++++++++ lib/images/errors.go | 6 +- lib/images/manager.go | 179 +++++++++++++++++++++++++++--- lib/images/manager_test.go | 218 +++++++++++++++++++++++++++++++++++++ lib/images/storage.go | 172 +++++++++++++++++++++++++++++ lib/providers/providers.go | 1 - 10 files changed, 816 insertions(+), 26 deletions(-) create mode 100644 lib/images/docker.go create mode 100644 lib/images/manager_test.go create mode 100644 lib/images/storage.go diff --git a/README.md b/README.md index 419c68a9..618f9d22 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,11 @@ cloud-hypervisor --version # Verify ch-remote --version ``` -**containerd** - [Installation guide](https://github.com/containerd/containerd/blob/main/docs/getting-started.md) +**Docker** - [Installation guide](https://docs.docker.com/engine/install/) ```bash -containerd --version # Verify +docker --version # Verify +# Add your user to docker group for non-root access: +sudo usermod -aG docker $USER ``` **Go 1.25.4+** and **KVM** diff --git a/cmd/api/api/api_test.go b/cmd/api/api/api_test.go index 305e6430..145bbb7e 100644 --- a/cmd/api/api/api_test.go +++ b/cmd/api/api/api_test.go @@ -27,4 +27,3 @@ func newTestService(t *testing.T) *ApiService { func ctx() context.Context { return context.Background() } - diff --git a/go.mod b/go.mod index ab83bfc6..0723f784 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/onkernel/hypeman go 1.25.4 require ( + github.com/docker/docker v28.5.2+incompatible github.com/getkin/kin-openapi v0.133.0 github.com/ghodss/yaml v1.0.0 github.com/go-chi/chi/v5 v5.2.3 @@ -16,20 +17,54 @@ require ( ) require ( + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/containerd/errdefs v0.3.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect - github.com/google/uuid v1.5.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.1.0 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/atomicwriter v0.1.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/morikuni/aec v1.0.0 // indirect github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/woodsbury/decimal128 v1.3.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/sdk v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/sys v0.31.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 2ced6493..6b867a28 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,45 @@ +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/containerd/errdefs v0.3.0 h1:FSZgGOeK4yuT/+DnF07/Olde/q4KBoMsaamhXxIMDp4= +github.com/containerd/errdefs v0.3.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= +github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/getkin/kin-openapi v0.133.0 h1:pJdmNohVIJ97r4AUFtEXRXwESr8b0bD721u/Tz6k8PQ= github.com/getkin/kin-openapi v0.133.0/go.mod h1:boAciF6cXk5FhPqe/NQeBTeenbjqU4LhWBf09ILVvWE= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE= github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= @@ -19,25 +48,49 @@ github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= +github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/oapi-codegen/nethttp-middleware v1.1.2 h1:TQwEU3WM6ifc7ObBEtiJgbRPaCe513tvJpiMJjypVPA= github.com/oapi-codegen/nethttp-middleware v1.1.2/go.mod h1:5qzjxMSiI8HjLljiOEjvs4RdrWyMPKnExeFS2kr8om4= github.com/oapi-codegen/runtime v1.1.2 h1:P2+CubHq8fO4Q6fV1tqDBZHCwpVpvPg7oKiYzQgXIyI= @@ -46,27 +99,74 @@ github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//J github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037/go.mod h1:2bpvgLBZEtENV5scfDFEtB/5+1M4hkQhDQrccEJ/qGw= github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 h1:bQx3WeLcUWy+RletIKwUIt4x3t8n2SxavmoclizMb8c= github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/woodsbury/decimal128 v1.3.0 h1:8pffMNWIlC0O5vbyHWFZAt5yWvWcrHA+3ovIIjVWss0= github.com/woodsbury/decimal128 v1.3.0/go.mod h1:C5UTmyTjW3JftjUFzOVhC20BEQa2a4ZKOB5I6Zjb+ds= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.67.0 h1:IdH9y6PF5MPSdAntIcpjQ+tXO41pcQsfZV2RxtQgVcw= +google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/lib/images/docker.go b/lib/images/docker.go new file mode 100644 index 00000000..a04056f8 --- /dev/null +++ b/lib/images/docker.go @@ -0,0 +1,114 @@ +package images + +import ( + "context" + "fmt" + "io" + + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/image" + "github.com/docker/docker/client" + "github.com/docker/docker/pkg/archive" +) + +// dockerClient wraps Docker API operations +type dockerClient struct { + cli *client.Client +} + +// newDockerClient creates a new Docker client using environment variables +func newDockerClient() (*dockerClient, error) { + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return nil, fmt.Errorf("create docker client: %w", err) + } + return &dockerClient{cli: cli}, nil +} + +// close closes the Docker client +func (d *dockerClient) close() error { + return d.cli.Close() +} + +// pullAndExport pulls an OCI image and exports its rootfs to a directory +func (d *dockerClient) pullAndExport(ctx context.Context, imageRef, exportDir string) (*containerMetadata, error) { + // Pull the image + pullReader, err := d.cli.ImagePull(ctx, imageRef, image.PullOptions{}) + if err != nil { + return nil, fmt.Errorf("pull image: %w", err) + } + // Consume the pull output (required for pull to complete) + io.Copy(io.Discard, pullReader) + pullReader.Close() + + // Inspect image to get metadata + inspect, _, err := d.cli.ImageInspectWithRaw(ctx, imageRef) + if err != nil { + return nil, fmt.Errorf("inspect image: %w", err) + } + + // Extract metadata from inspect response + meta := &containerMetadata{ + Entrypoint: inspect.Config.Entrypoint, + Cmd: inspect.Config.Cmd, + Env: make(map[string]string), + WorkingDir: inspect.Config.WorkingDir, + } + + // Parse environment variables + for _, env := range inspect.Config.Env { + for i := 0; i < len(env); i++ { + if env[i] == '=' { + key := env[:i] + val := env[i+1:] + meta.Env[key] = val + break + } + } + } + + // Create a temporary container from the image + // Creating a container does not run it, + // it does not execute any code inside the container. + containerResp, err := d.cli.ContainerCreate(ctx, &container.Config{ + Image: imageRef, + }, nil, nil, nil, "") + if err != nil { + return nil, fmt.Errorf("create container: %w", err) + } + + // Ensure container is removed even if export fails + defer func() { + d.cli.ContainerRemove(ctx, containerResp.ID, container.RemoveOptions{}) + }() + + // Export container filesystem as tar stream + exportReader, err := d.cli.ContainerExport(ctx, containerResp.ID) + if err != nil { + return nil, fmt.Errorf("export container: %w", err) + } + defer exportReader.Close() + + // Extract tar to exportDir + if err := extractTar(exportReader, exportDir); err != nil { + return nil, fmt.Errorf("extract tar: %w", err) + } + + return meta, nil +} + +// containerMetadata holds extracted container metadata +type containerMetadata struct { + Entrypoint []string + Cmd []string + Env map[string]string + WorkingDir string +} + +// extractTar extracts a tar stream to a target directory using Docker's archive package +func extractTar(reader io.Reader, targetDir string) error { + // Use Docker's battle-tested archive extraction with security hardening + return archive.Untar(reader, targetDir, &archive.TarOptions{ + NoLchown: true, // Don't change ownership (we're not root) + }) +} diff --git a/lib/images/errors.go b/lib/images/errors.go index 29baf9e9..030a9c4f 100644 --- a/lib/images/errors.go +++ b/lib/images/errors.go @@ -3,7 +3,9 @@ package images import "errors" var ( - ErrNotFound = errors.New("image not found") - ErrAlreadyExists = errors.New("image already exists") + ErrNotFound = errors.New("image not found") + ErrAlreadyExists = errors.New("image already exists") + ErrInvalidImage = errors.New("invalid image reference") + ErrConversionFailed = errors.New("failed to convert image") ) diff --git a/lib/images/manager.go b/lib/images/manager.go index 0f7f2104..bd83e0cd 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -2,7 +2,14 @@ package images import ( "context" + "crypto/sha256" "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "time" "github.com/onkernel/hypeman/lib/oapi" ) @@ -27,32 +34,174 @@ func NewManager(dataDir string) Manager { } func (m *manager) ListImages(ctx context.Context) ([]oapi.Image, error) { - // TODO: implement - return []oapi.Image{}, nil + metas, err := listMetadata(m.dataDir) + if err != nil { + return nil, fmt.Errorf("list metadata: %w", err) + } + + images := make([]oapi.Image, 0, len(metas)) + for _, meta := range metas { + images = append(images, *meta.toOAPI()) + } + + return images, nil } func (m *manager) CreateImage(ctx context.Context, req oapi.CreateImageRequest) (*oapi.Image, error) { - // TODO: implement actual logic - // Example: check if already exists - exists := false - if exists { + // 1. Generate or validate ID + imageID := req.Id + if imageID == nil || *imageID == "" { + generated := generateImageID(req.Name) + imageID = &generated + } + + // 2. Check if image already exists + if imageExists(m.dataDir, *imageID) { return nil, ErrAlreadyExists } - return nil, fmt.Errorf("image creation not yet implemented") + + // 3. Connect to Docker + client, err := newDockerClient() + if err != nil { + return nil, fmt.Errorf("connect to docker: %w", err) + } + defer client.close() + + // 4. Pull image and export rootfs to temp directory + tempDir := filepath.Join(os.TempDir(), fmt.Sprintf("hypeman-image-%s-%d", *imageID, time.Now().Unix())) + defer os.RemoveAll(tempDir) // cleanup temp dir + + containerMeta, err := client.pullAndExport(ctx, req.Name, tempDir) + if err != nil { + return nil, fmt.Errorf("pull and export: %w", err) + } + + // 5. Convert rootfs directory to ext4 disk image + diskPath := imagePath(m.dataDir, *imageID) + diskSize, err := convertToExt4(tempDir, diskPath) + if err != nil { + return nil, fmt.Errorf("convert to ext4: %w", err) + } + + // 6. Create metadata + meta := &imageMetadata{ + ID: *imageID, + Name: req.Name, + SizeBytes: diskSize, + Entrypoint: containerMeta.Entrypoint, + Cmd: containerMeta.Cmd, + Env: containerMeta.Env, + WorkingDir: containerMeta.WorkingDir, + CreatedAt: time.Now(), + } + + // 7. Write metadata atomically + if err := writeMetadata(m.dataDir, *imageID, meta); err != nil { + // Clean up disk image if metadata write fails + os.Remove(diskPath) + return nil, fmt.Errorf("write metadata: %w", err) + } + + return meta.toOAPI(), nil } func (m *manager) GetImage(ctx context.Context, id string) (*oapi.Image, error) { - // TODO: implement actual logic - // For now, always return not found since we have no images - return nil, ErrNotFound + meta, err := readMetadata(m.dataDir, id) + if err != nil { + return nil, err + } + return meta.toOAPI(), nil } func (m *manager) DeleteImage(ctx context.Context, id string) error { - // TODO: implement actual logic - exists := false - if !exists { - return ErrNotFound + return deleteImage(m.dataDir, id) +} + +// generateImageID creates a valid ID from an image name +// Example: docker.io/library/nginx:latest -> img-nginx-latest +func generateImageID(imageName string) string { + // Extract image name and tag + parts := strings.Split(imageName, "/") + nameTag := parts[len(parts)-1] + + // Replace special characters with dashes + reg := regexp.MustCompile(`[^a-zA-Z0-9]+`) + sanitized := reg.ReplaceAllString(nameTag, "-") + sanitized = strings.Trim(sanitized, "-") + + // Add prefix + return "img-" + sanitized +} + +// convertToExt4 converts a rootfs directory to an ext4 disk image +func convertToExt4(rootfsDir, diskPath string) (int64, error) { + // Calculate size of rootfs directory (rounded up to nearest GB, minimum 1GB) + sizeBytes, err := dirSize(rootfsDir) + if err != nil { + return 0, fmt.Errorf("calculate dir size: %w", err) + } + + // Add 20% overhead for filesystem metadata, minimum 1GB + diskSizeBytes := sizeBytes + (sizeBytes / 5) + const minSize = 1024 * 1024 * 1024 // 1GB + if diskSizeBytes < minSize { + diskSizeBytes = minSize + } + + // Ensure parent directory exists + if err := os.MkdirAll(filepath.Dir(diskPath), 0755); err != nil { + return 0, fmt.Errorf("create disk parent dir: %w", err) + } + + // Create sparse file + f, err := os.Create(diskPath) + if err != nil { + return 0, fmt.Errorf("create disk file: %w", err) + } + if err := f.Truncate(diskSizeBytes); err != nil { + f.Close() + return 0, fmt.Errorf("truncate disk file: %w", err) + } + f.Close() + + // Format as ext4 with rootfs contents + cmd := exec.Command("mkfs.ext4", "-d", rootfsDir, "-F", diskPath) + output, err := cmd.CombinedOutput() + if err != nil { + return 0, fmt.Errorf("mkfs.ext4 failed: %w, output: %s", err, output) + } + + // Get actual disk size + stat, err := os.Stat(diskPath) + if err != nil { + return 0, fmt.Errorf("stat disk: %w", err) + } + + return stat.Size(), nil +} + +// dirSize calculates the total size of a directory +func dirSize(path string) (int64, error) { + var size int64 + err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + size += info.Size() + } + return nil + }) + return size, err +} + +// checksumFile computes sha256 of a file +func checksumFile(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err } - return fmt.Errorf("delete image not yet implemented") + hash := sha256.Sum256(data) + return fmt.Sprintf("%x", hash), nil } diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go new file mode 100644 index 00000000..1d72022c --- /dev/null +++ b/lib/images/manager_test.go @@ -0,0 +1,218 @@ +package images + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/onkernel/hypeman/lib/oapi" + "github.com/stretchr/testify/require" +) + +func TestCreateImage(t *testing.T) { + skipIfNoDocker(t) + + dataDir := t.TempDir() + mgr := NewManager(dataDir) + + ctx := context.Background() + req := oapi.CreateImageRequest{ + Name: "docker.io/library/alpine:latest", + } + + img, err := mgr.CreateImage(ctx, req) + require.NoError(t, err) + require.NotNil(t, img) + require.Equal(t, "docker.io/library/alpine:latest", img.Name) + require.Equal(t, "img-alpine-latest", img.Id) + require.NotNil(t, img.SizeBytes) + require.Greater(t, *img.SizeBytes, int64(0)) + + // Verify disk image was created + diskPath := filepath.Join(dataDir, "images", img.Id, "rootfs.ext4") + _, err = os.Stat(diskPath) + require.NoError(t, err) + + // Verify metadata file was created + metaPath := filepath.Join(dataDir, "images", img.Id, "metadata.json") + _, err = os.Stat(metaPath) + require.NoError(t, err) +} + +func TestCreateImageWithCustomID(t *testing.T) { + skipIfNoDocker(t) + + dataDir := t.TempDir() + mgr := NewManager(dataDir) + + ctx := context.Background() + customID := "my-custom-alpine" + req := oapi.CreateImageRequest{ + Name: "docker.io/library/alpine:latest", + Id: &customID, + } + + img, err := mgr.CreateImage(ctx, req) + require.NoError(t, err) + require.NotNil(t, img) + require.Equal(t, "my-custom-alpine", img.Id) +} + +func TestCreateImageDuplicate(t *testing.T) { + skipIfNoDocker(t) + + dataDir := t.TempDir() + mgr := NewManager(dataDir) + + ctx := context.Background() + req := oapi.CreateImageRequest{ + Name: "docker.io/library/alpine:latest", + } + + // Create first image + _, err := mgr.CreateImage(ctx, req) + require.NoError(t, err) + + // Try to create duplicate + _, err = mgr.CreateImage(ctx, req) + require.ErrorIs(t, err, ErrAlreadyExists) +} + +func TestListImages(t *testing.T) { + skipIfNoDocker(t) + + dataDir := t.TempDir() + mgr := NewManager(dataDir) + + ctx := context.Background() + + // Initially empty + images, err := mgr.ListImages(ctx) + require.NoError(t, err) + require.Len(t, images, 0) + + // Create first image + req1 := oapi.CreateImageRequest{ + Name: "docker.io/library/alpine:latest", + } + _, err = mgr.CreateImage(ctx, req1) + require.NoError(t, err) + + // List should return one image + images, err = mgr.ListImages(ctx) + require.NoError(t, err) + require.Len(t, images, 1) + require.Equal(t, "docker.io/library/alpine:latest", images[0].Name) +} + +func TestGetImage(t *testing.T) { + skipIfNoDocker(t) + + dataDir := t.TempDir() + mgr := NewManager(dataDir) + + ctx := context.Background() + req := oapi.CreateImageRequest{ + Name: "docker.io/library/alpine:latest", + } + + created, err := mgr.CreateImage(ctx, req) + require.NoError(t, err) + + // Get the image + img, err := mgr.GetImage(ctx, created.Id) + require.NoError(t, err) + require.NotNil(t, img) + require.Equal(t, created.Id, img.Id) + require.Equal(t, created.Name, img.Name) + require.Equal(t, *created.SizeBytes, *img.SizeBytes) +} + +func TestGetImageNotFound(t *testing.T) { + dataDir := t.TempDir() + mgr := NewManager(dataDir) + + ctx := context.Background() + + // Try to get non-existent image + _, err := mgr.GetImage(ctx, "nonexistent") + require.ErrorIs(t, err, ErrNotFound) +} + +func TestDeleteImage(t *testing.T) { + skipIfNoDocker(t) + + dataDir := t.TempDir() + mgr := NewManager(dataDir) + + ctx := context.Background() + req := oapi.CreateImageRequest{ + Name: "docker.io/library/alpine:latest", + } + + created, err := mgr.CreateImage(ctx, req) + require.NoError(t, err) + + // Delete the image + err = mgr.DeleteImage(ctx, created.Id) + require.NoError(t, err) + + // Verify it's gone + _, err = mgr.GetImage(ctx, created.Id) + require.ErrorIs(t, err, ErrNotFound) + + // Verify files were removed + imageDir := filepath.Join(dataDir, "images", created.Id) + _, err = os.Stat(imageDir) + require.True(t, os.IsNotExist(err)) +} + +func TestDeleteImageNotFound(t *testing.T) { + dataDir := t.TempDir() + mgr := NewManager(dataDir) + + ctx := context.Background() + + // Try to delete non-existent image + err := mgr.DeleteImage(ctx, "nonexistent") + require.ErrorIs(t, err, ErrNotFound) +} + +func TestGenerateImageID(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"docker.io/library/nginx:latest", "img-nginx-latest"}, + {"docker.io/library/alpine:3.18", "img-alpine-3-18"}, + {"gcr.io/my-project/my-app:v1.0.0", "img-my-app-v1-0-0"}, + {"nginx", "img-nginx"}, + {"ubuntu:22.04", "img-ubuntu-22-04"}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result := generateImageID(tt.input) + require.Equal(t, tt.expected, result) + }) + } +} + +// skipIfNoDocker skips the test if Docker is not available or accessible +func skipIfNoDocker(t *testing.T) { + // Try to connect to Docker to verify we have permission + client, err := newDockerClient() + if err != nil { + t.Skipf("cannot connect to docker: %v, skipping test", err) + } + defer client.close() + + // Verify we can actually use Docker by pinging it + ctx := context.Background() + _, err = client.cli.Ping(ctx) + if err != nil { + t.Skipf("docker not available: %v, skipping test", err) + } +} + diff --git a/lib/images/storage.go b/lib/images/storage.go new file mode 100644 index 00000000..9408383c --- /dev/null +++ b/lib/images/storage.go @@ -0,0 +1,172 @@ +package images + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/onkernel/hypeman/lib/oapi" +) + +// imageMetadata represents the metadata stored on disk +type imageMetadata struct { + ID string `json:"id"` + Name string `json:"name"` + SizeBytes int64 `json:"size_bytes"` + Entrypoint []string `json:"entrypoint,omitempty"` + Cmd []string `json:"cmd,omitempty"` + Env map[string]string `json:"env,omitempty"` + WorkingDir string `json:"working_dir,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// toOAPI converts internal metadata to OpenAPI schema +func (m *imageMetadata) toOAPI() *oapi.Image { + sizeBytes := m.SizeBytes + img := &oapi.Image{ + Id: m.ID, + Name: m.Name, + SizeBytes: &sizeBytes, + CreatedAt: m.CreatedAt, + } + + if len(m.Entrypoint) > 0 { + img.Entrypoint = &m.Entrypoint + } + if len(m.Cmd) > 0 { + img.Cmd = &m.Cmd + } + if len(m.Env) > 0 { + img.Env = &m.Env + } + if m.WorkingDir != "" { + img.WorkingDir = &m.WorkingDir + } + + return img +} + +// imageDir returns the directory path for an image +func imageDir(dataDir, imageID string) string { + return filepath.Join(dataDir, "images", imageID) +} + +// imagePath returns the path to the rootfs disk image +func imagePath(dataDir, imageID string) string { + return filepath.Join(imageDir(dataDir, imageID), "rootfs.ext4") +} + +// metadataPath returns the path to the metadata file +func metadataPath(dataDir, imageID string) string { + return filepath.Join(imageDir(dataDir, imageID), "metadata.json") +} + +// writeMetadata writes metadata atomically using temp file + rename +func writeMetadata(dataDir, imageID string, meta *imageMetadata) error { + dir := imageDir(dataDir, imageID) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("create image directory: %w", err) + } + + data, err := json.MarshalIndent(meta, "", " ") + if err != nil { + return fmt.Errorf("marshal metadata: %w", err) + } + + // Write to temp file first + tempPath := metadataPath(dataDir, imageID) + ".tmp" + if err := os.WriteFile(tempPath, data, 0644); err != nil { + return fmt.Errorf("write temp metadata: %w", err) + } + + // Atomic rename + finalPath := metadataPath(dataDir, imageID) + if err := os.Rename(tempPath, finalPath); err != nil { + os.Remove(tempPath) // cleanup + return fmt.Errorf("rename metadata: %w", err) + } + + return nil +} + +// readMetadata reads metadata from disk +func readMetadata(dataDir, imageID string) (*imageMetadata, error) { + path := metadataPath(dataDir, imageID) + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("read metadata: %w", err) + } + + var meta imageMetadata + if err := json.Unmarshal(data, &meta); err != nil { + return nil, fmt.Errorf("unmarshal metadata: %w", err) + } + + // Validate that disk image exists + diskPath := imagePath(dataDir, imageID) + if _, err := os.Stat(diskPath); err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("disk image missing: %s", diskPath) + } + return nil, fmt.Errorf("stat disk image: %w", err) + } + + return &meta, nil +} + +// listMetadata lists all image metadata by scanning the images directory +func listMetadata(dataDir string) ([]*imageMetadata, error) { + imagesDir := filepath.Join(dataDir, "images") + entries, err := os.ReadDir(imagesDir) + if err != nil { + if os.IsNotExist(err) { + return []*imageMetadata{}, nil + } + return nil, fmt.Errorf("read images directory: %w", err) + } + + var metas []*imageMetadata + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + meta, err := readMetadata(dataDir, entry.Name()) + if err != nil { + // Skip invalid entries, log but don't fail + continue + } + metas = append(metas, meta) + } + + return metas, nil +} + +// imageExists checks if an image already exists +func imageExists(dataDir, imageID string) bool { + _, err := readMetadata(dataDir, imageID) + return err == nil +} + +// deleteImage removes the entire image directory +func deleteImage(dataDir, imageID string) error { + dir := imageDir(dataDir, imageID) + if _, err := os.Stat(dir); err != nil { + if os.IsNotExist(err) { + return ErrNotFound + } + return fmt.Errorf("stat image directory: %w", err) + } + + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("remove image directory: %w", err) + } + + return nil +} + diff --git a/lib/providers/providers.go b/lib/providers/providers.go index ea1fd2dc..8c46e7ec 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -43,4 +43,3 @@ func ProvideInstanceManager(cfg *config.Config) instances.Manager { func ProvideVolumeManager(cfg *config.Config) volumes.Manager { return volumes.NewManager(cfg.DataDir) } - From 521917dab299e00a4182fd0d67f3c9bd1bd51821 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Thu, 6 Nov 2025 18:10:20 -0500 Subject: [PATCH 02/37] docker client dependency injection --- cmd/api/api/api_test.go | 5 +++- cmd/api/config/config.go | 30 ++++++++++++------------ cmd/api/wire.go | 1 + cmd/api/wire_gen.go | 6 ++++- lib/images/docker.go | 16 ++++++------- lib/images/manager.go | 21 +++++++---------- lib/images/manager_test.go | 47 ++++++++++++++++++++++++-------------- lib/providers/providers.go | 9 ++++++-- 8 files changed, 77 insertions(+), 58 deletions(-) diff --git a/cmd/api/api/api_test.go b/cmd/api/api/api_test.go index 145bbb7e..10e92caa 100644 --- a/cmd/api/api/api_test.go +++ b/cmd/api/api/api_test.go @@ -16,9 +16,12 @@ func newTestService(t *testing.T) *ApiService { DataDir: t.TempDir(), } + // Create Docker client for testing (may be nil if not available) + dockerClient, _ := images.NewDockerClient() + return &ApiService{ Config: cfg, - ImageManager: images.NewManager(cfg.DataDir), + ImageManager: images.NewManager(cfg.DataDir, dockerClient), InstanceManager: instances.NewManager(cfg.DataDir), VolumeManager: volumes.NewManager(cfg.DataDir), } diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 9ed6451b..99c441ac 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -7,14 +7,13 @@ import ( ) type Config struct { - Port string - DataDir string - BridgeName string - SubnetCIDR string - SubnetGateway string - ContainerdSocket string - JwtSecret string - DNSServer string + Port string + DataDir string + BridgeName string + SubnetCIDR string + SubnetGateway string + JwtSecret string + DNSServer string } // Load loads configuration from environment variables @@ -24,14 +23,13 @@ func Load() *Config { _ = godotenv.Load() cfg := &Config{ - Port: getEnv("PORT", "8080"), - DataDir: getEnv("DATA_DIR", "/var/lib/hypeman"), - BridgeName: getEnv("BRIDGE_NAME", "vmbr0"), - SubnetCIDR: getEnv("SUBNET_CIDR", "192.168.100.0/24"), - SubnetGateway: getEnv("SUBNET_GATEWAY", "192.168.100.1"), - ContainerdSocket: getEnv("CONTAINERD_SOCKET", "/run/containerd/containerd.sock"), - JwtSecret: getEnv("JWT_SECRET", ""), - DNSServer: getEnv("DNS_SERVER", "1.1.1.1"), + Port: getEnv("PORT", "8080"), + DataDir: getEnv("DATA_DIR", "/var/lib/hypeman"), + BridgeName: getEnv("BRIDGE_NAME", "vmbr0"), + SubnetCIDR: getEnv("SUBNET_CIDR", "192.168.100.0/24"), + SubnetGateway: getEnv("SUBNET_GATEWAY", "192.168.100.1"), + JwtSecret: getEnv("JWT_SECRET", ""), + DNSServer: getEnv("DNS_SERVER", "1.1.1.1"), } return cfg diff --git a/cmd/api/wire.go b/cmd/api/wire.go index f9678ba6..e422c548 100644 --- a/cmd/api/wire.go +++ b/cmd/api/wire.go @@ -32,6 +32,7 @@ func initializeApp() (*application, func(), error) { providers.ProvideLogger, providers.ProvideContext, providers.ProvideConfig, + providers.ProvideDockerClient, providers.ProvideImageManager, providers.ProvideInstanceManager, providers.ProvideVolumeManager, diff --git a/cmd/api/wire_gen.go b/cmd/api/wire_gen.go index 1fea4142..0ef19e26 100644 --- a/cmd/api/wire_gen.go +++ b/cmd/api/wire_gen.go @@ -28,7 +28,11 @@ func initializeApp() (*application, func(), error) { logger := providers.ProvideLogger() context := providers.ProvideContext(logger) config := providers.ProvideConfig() - manager := providers.ProvideImageManager(config) + dockerClient, err := providers.ProvideDockerClient() + if err != nil { + return nil, nil, err + } + manager := providers.ProvideImageManager(config, dockerClient) instancesManager := providers.ProvideInstanceManager(config) volumesManager := providers.ProvideVolumeManager(config) apiService := api.New(config, manager, instancesManager, volumesManager) diff --git a/lib/images/docker.go b/lib/images/docker.go index a04056f8..51f9f6eb 100644 --- a/lib/images/docker.go +++ b/lib/images/docker.go @@ -11,27 +11,27 @@ import ( "github.com/docker/docker/pkg/archive" ) -// dockerClient wraps Docker API operations -type dockerClient struct { +// DockerClient wraps Docker API operations +type DockerClient struct { cli *client.Client } -// newDockerClient creates a new Docker client using environment variables -func newDockerClient() (*dockerClient, error) { +// NewDockerClient creates a new Docker client using environment variables +func NewDockerClient() (*DockerClient, error) { cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { return nil, fmt.Errorf("create docker client: %w", err) } - return &dockerClient{cli: cli}, nil + return &DockerClient{cli: cli}, nil } -// close closes the Docker client -func (d *dockerClient) close() error { +// Close closes the Docker client +func (d *DockerClient) Close() error { return d.cli.Close() } // pullAndExport pulls an OCI image and exports its rootfs to a directory -func (d *dockerClient) pullAndExport(ctx context.Context, imageRef, exportDir string) (*containerMetadata, error) { +func (d *DockerClient) pullAndExport(ctx context.Context, imageRef, exportDir string) (*containerMetadata, error) { // Pull the image pullReader, err := d.cli.ImagePull(ctx, imageRef, image.PullOptions{}) if err != nil { diff --git a/lib/images/manager.go b/lib/images/manager.go index bd83e0cd..ba3faf3b 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -23,13 +23,15 @@ type Manager interface { } type manager struct { - dataDir string + dataDir string + dockerClient *DockerClient } -// NewManager creates a new image manager -func NewManager(dataDir string) Manager { +// NewManager creates a new image manager with Docker client +func NewManager(dataDir string, dockerClient *DockerClient) Manager { return &manager{ - dataDir: dataDir, + dataDir: dataDir, + dockerClient: dockerClient, } } @@ -60,18 +62,11 @@ func (m *manager) CreateImage(ctx context.Context, req oapi.CreateImageRequest) return nil, ErrAlreadyExists } - // 3. Connect to Docker - client, err := newDockerClient() - if err != nil { - return nil, fmt.Errorf("connect to docker: %w", err) - } - defer client.close() - - // 4. Pull image and export rootfs to temp directory + // 3. Pull image and export rootfs to temp directory tempDir := filepath.Join(os.TempDir(), fmt.Sprintf("hypeman-image-%s-%d", *imageID, time.Now().Unix())) defer os.RemoveAll(tempDir) // cleanup temp dir - containerMeta, err := client.pullAndExport(ctx, req.Name, tempDir) + containerMeta, err := m.dockerClient.pullAndExport(ctx, req.Name, tempDir) if err != nil { return nil, fmt.Errorf("pull and export: %w", err) } diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go index 1d72022c..7c306d87 100644 --- a/lib/images/manager_test.go +++ b/lib/images/manager_test.go @@ -11,10 +11,10 @@ import ( ) func TestCreateImage(t *testing.T) { - skipIfNoDocker(t) + dockerClient := skipIfNoDocker(t) dataDir := t.TempDir() - mgr := NewManager(dataDir) + mgr := NewManager(dataDir, dockerClient) ctx := context.Background() req := oapi.CreateImageRequest{ @@ -41,10 +41,10 @@ func TestCreateImage(t *testing.T) { } func TestCreateImageWithCustomID(t *testing.T) { - skipIfNoDocker(t) + dockerClient := skipIfNoDocker(t) dataDir := t.TempDir() - mgr := NewManager(dataDir) + mgr := NewManager(dataDir, dockerClient) ctx := context.Background() customID := "my-custom-alpine" @@ -60,10 +60,10 @@ func TestCreateImageWithCustomID(t *testing.T) { } func TestCreateImageDuplicate(t *testing.T) { - skipIfNoDocker(t) + dockerClient := skipIfNoDocker(t) dataDir := t.TempDir() - mgr := NewManager(dataDir) + mgr := NewManager(dataDir, dockerClient) ctx := context.Background() req := oapi.CreateImageRequest{ @@ -80,10 +80,10 @@ func TestCreateImageDuplicate(t *testing.T) { } func TestListImages(t *testing.T) { - skipIfNoDocker(t) + dockerClient := skipIfNoDocker(t) dataDir := t.TempDir() - mgr := NewManager(dataDir) + mgr := NewManager(dataDir, dockerClient) ctx := context.Background() @@ -107,10 +107,10 @@ func TestListImages(t *testing.T) { } func TestGetImage(t *testing.T) { - skipIfNoDocker(t) + dockerClient := skipIfNoDocker(t) dataDir := t.TempDir() - mgr := NewManager(dataDir) + mgr := NewManager(dataDir, dockerClient) ctx := context.Background() req := oapi.CreateImageRequest{ @@ -130,8 +130,13 @@ func TestGetImage(t *testing.T) { } func TestGetImageNotFound(t *testing.T) { + dockerClient, _ := NewDockerClient() + if dockerClient != nil { + defer dockerClient.Close() + } + dataDir := t.TempDir() - mgr := NewManager(dataDir) + mgr := NewManager(dataDir, dockerClient) ctx := context.Background() @@ -141,10 +146,10 @@ func TestGetImageNotFound(t *testing.T) { } func TestDeleteImage(t *testing.T) { - skipIfNoDocker(t) + dockerClient := skipIfNoDocker(t) dataDir := t.TempDir() - mgr := NewManager(dataDir) + mgr := NewManager(dataDir, dockerClient) ctx := context.Background() req := oapi.CreateImageRequest{ @@ -169,8 +174,13 @@ func TestDeleteImage(t *testing.T) { } func TestDeleteImageNotFound(t *testing.T) { + dockerClient, _ := NewDockerClient() + if dockerClient != nil { + defer dockerClient.Close() + } + dataDir := t.TempDir() - mgr := NewManager(dataDir) + mgr := NewManager(dataDir, dockerClient) ctx := context.Background() @@ -200,19 +210,22 @@ func TestGenerateImageID(t *testing.T) { } // skipIfNoDocker skips the test if Docker is not available or accessible -func skipIfNoDocker(t *testing.T) { +// Returns a DockerClient for use in tests +func skipIfNoDocker(t *testing.T) *DockerClient { // Try to connect to Docker to verify we have permission - client, err := newDockerClient() + client, err := NewDockerClient() if err != nil { t.Skipf("cannot connect to docker: %v, skipping test", err) } - defer client.close() // Verify we can actually use Docker by pinging it ctx := context.Background() _, err = client.cli.Ping(ctx) if err != nil { + client.Close() t.Skipf("docker not available: %v, skipping test", err) } + + return client } diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 8c46e7ec..3d7f2cc7 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -29,9 +29,14 @@ func ProvideConfig() *config.Config { return config.Load() } +// ProvideDockerClient provides a Docker client +func ProvideDockerClient() (*images.DockerClient, error) { + return images.NewDockerClient() +} + // ProvideImageManager provides the image manager -func ProvideImageManager(cfg *config.Config) images.Manager { - return images.NewManager(cfg.DataDir) +func ProvideImageManager(cfg *config.Config, dockerClient *images.DockerClient) images.Manager { + return images.NewManager(cfg.DataDir, dockerClient) } // ProvideInstanceManager provides the instance manager From 1b7b6a390d110b93f71e70cdedf355c55512ecf4 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Thu, 6 Nov 2025 18:25:25 -0500 Subject: [PATCH 03/37] Fix permissions during extraction --- lib/images/docker.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/images/docker.go b/lib/images/docker.go index 51f9f6eb..8d0ff813 100644 --- a/lib/images/docker.go +++ b/lib/images/docker.go @@ -4,11 +4,13 @@ import ( "context" "fmt" "io" + "os" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" "github.com/docker/docker/client" "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/idtools" ) // DockerClient wraps Docker API operations @@ -108,7 +110,13 @@ type containerMetadata struct { // extractTar extracts a tar stream to a target directory using Docker's archive package func extractTar(reader io.Reader, targetDir string) error { // Use Docker's battle-tested archive extraction with security hardening + // Set ownership to current user instead of trying to preserve original ownership return archive.Untar(reader, targetDir, &archive.TarOptions{ - NoLchown: true, // Don't change ownership (we're not root) + NoLchown: true, + ChownOpts: &idtools.Identity{ + UID: os.Getuid(), + GID: os.Getgid(), + }, + InUserNS: true, // Skip chown operations (we're in user namespace / not root) }) } From e0110bd4677ca70ddc984317f8939b0ae387e3ac Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Thu, 6 Nov 2025 18:27:37 -0500 Subject: [PATCH 04/37] Don't skip docker in tests --- lib/images/manager_test.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go index 7c306d87..d52e9e9f 100644 --- a/lib/images/manager_test.go +++ b/lib/images/manager_test.go @@ -11,7 +11,7 @@ import ( ) func TestCreateImage(t *testing.T) { - dockerClient := skipIfNoDocker(t) + dockerClient := requireDocker(t) dataDir := t.TempDir() mgr := NewManager(dataDir, dockerClient) @@ -41,7 +41,7 @@ func TestCreateImage(t *testing.T) { } func TestCreateImageWithCustomID(t *testing.T) { - dockerClient := skipIfNoDocker(t) + dockerClient := requireDocker(t) dataDir := t.TempDir() mgr := NewManager(dataDir, dockerClient) @@ -60,7 +60,7 @@ func TestCreateImageWithCustomID(t *testing.T) { } func TestCreateImageDuplicate(t *testing.T) { - dockerClient := skipIfNoDocker(t) + dockerClient := requireDocker(t) dataDir := t.TempDir() mgr := NewManager(dataDir, dockerClient) @@ -80,7 +80,7 @@ func TestCreateImageDuplicate(t *testing.T) { } func TestListImages(t *testing.T) { - dockerClient := skipIfNoDocker(t) + dockerClient := requireDocker(t) dataDir := t.TempDir() mgr := NewManager(dataDir, dockerClient) @@ -107,7 +107,7 @@ func TestListImages(t *testing.T) { } func TestGetImage(t *testing.T) { - dockerClient := skipIfNoDocker(t) + dockerClient := requireDocker(t) dataDir := t.TempDir() mgr := NewManager(dataDir, dockerClient) @@ -146,7 +146,7 @@ func TestGetImageNotFound(t *testing.T) { } func TestDeleteImage(t *testing.T) { - dockerClient := skipIfNoDocker(t) + dockerClient := requireDocker(t) dataDir := t.TempDir() mgr := NewManager(dataDir, dockerClient) @@ -209,13 +209,13 @@ func TestGenerateImageID(t *testing.T) { } } -// skipIfNoDocker skips the test if Docker is not available or accessible +// requireDocker fails the test if Docker is not available or accessible // Returns a DockerClient for use in tests -func skipIfNoDocker(t *testing.T) *DockerClient { +func requireDocker(t *testing.T) *DockerClient { // Try to connect to Docker to verify we have permission client, err := NewDockerClient() if err != nil { - t.Skipf("cannot connect to docker: %v, skipping test", err) + t.Fatalf("cannot connect to docker: %v", err) } // Verify we can actually use Docker by pinging it @@ -223,7 +223,7 @@ func skipIfNoDocker(t *testing.T) *DockerClient { _, err = client.cli.Ping(ctx) if err != nil { client.Close() - t.Skipf("docker not available: %v, skipping test", err) + t.Fatalf("docker not available: %v", err) } return client From 817a85ce98063490d7455715e94f656b44886214 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Thu, 6 Nov 2025 18:37:49 -0500 Subject: [PATCH 05/37] extraction test passing but seems too complicated --- lib/images/docker.go | 63 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 13 deletions(-) diff --git a/lib/images/docker.go b/lib/images/docker.go index 8d0ff813..a91c66ab 100644 --- a/lib/images/docker.go +++ b/lib/images/docker.go @@ -1,16 +1,16 @@ package images import ( + "archive/tar" "context" "fmt" "io" "os" + "path/filepath" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" "github.com/docker/docker/client" - "github.com/docker/docker/pkg/archive" - "github.com/docker/docker/pkg/idtools" ) // DockerClient wraps Docker API operations @@ -107,16 +107,53 @@ type containerMetadata struct { WorkingDir string } -// extractTar extracts a tar stream to a target directory using Docker's archive package +// extractTar extracts a tar stream to a target directory +// Ignores ownership/permission errors (matches POC behavior: docker export | tar -C dir -xf -) func extractTar(reader io.Reader, targetDir string) error { - // Use Docker's battle-tested archive extraction with security hardening - // Set ownership to current user instead of trying to preserve original ownership - return archive.Untar(reader, targetDir, &archive.TarOptions{ - NoLchown: true, - ChownOpts: &idtools.Identity{ - UID: os.Getuid(), - GID: os.Getgid(), - }, - InUserNS: true, // Skip chown operations (we're in user namespace / not root) - }) + tarReader := tar.NewReader(reader) + + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("read tar header: %w", err) + } + + target := filepath.Join(targetDir, header.Name) + + switch header.Typeflag { + case tar.TypeDir: + // Create directory, ignore chmod errors (best effort) + os.MkdirAll(target, 0755) + + case tar.TypeReg: + // Create parent directory + os.MkdirAll(filepath.Dir(target), 0755) + + // Create file, use default permissions if header.Mode fails + outFile, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("create file %s: %w", target, err) + } + if _, err := io.Copy(outFile, tarReader); err != nil { + outFile.Close() + return fmt.Errorf("write file %s: %w", target, err) + } + outFile.Close() + + case tar.TypeSymlink: + os.MkdirAll(filepath.Dir(target), 0755) + os.Remove(target) // Remove if exists + os.Symlink(header.Linkname, target) // Ignore errors + + case tar.TypeLink: + linkTarget := filepath.Join(targetDir, header.Linkname) + os.Remove(target) + os.Link(linkTarget, target) // Ignore errors + } + } + + return nil } From 19e4c644b3ce6301775c31cf9ebeb673059b1922 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 7 Nov 2025 10:33:02 -0500 Subject: [PATCH 06/37] Use umoci for image to rootfs conversion --- Makefile | 4 +- cmd/api/api/api_test.go | 9 +- cmd/api/wire.go | 2 +- cmd/api/wire_gen.go | 4 +- go.mod | 89 ++++++---- go.sum | 340 +++++++++++++++++++++++++++++-------- lib/images/disk.go | 72 ++++++++ lib/images/docker.go | 159 ----------------- lib/images/manager.go | 87 +--------- lib/images/manager_test.go | 71 +++----- lib/images/oci.go | 240 ++++++++++++++++++++++++++ lib/providers/providers.go | 12 +- 12 files changed, 697 insertions(+), 392 deletions(-) create mode 100644 lib/images/disk.go delete mode 100644 lib/images/docker.go create mode 100644 lib/images/oci.go diff --git a/Makefile b/Makefile index 94a3204c..0412ccc5 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ generate-all: oapi-generate generate-wire # Build the binary build: | $(BIN_DIR) - go build -o $(BIN_DIR)/hypeman ./cmd/api + go build -tags containers_image_openpgp -o $(BIN_DIR)/hypeman ./cmd/api # Run in development mode with hot reload dev: $(AIR) @@ -51,7 +51,7 @@ dev: $(AIR) # Run tests test: - go test -v -timeout 30s ./... + go test -tags containers_image_openpgp -v -timeout 30s ./... # Clean generated files and binaries clean: diff --git a/cmd/api/api/api_test.go b/cmd/api/api/api_test.go index 10e92caa..6328b61b 100644 --- a/cmd/api/api/api_test.go +++ b/cmd/api/api/api_test.go @@ -16,12 +16,15 @@ func newTestService(t *testing.T) *ApiService { DataDir: t.TempDir(), } - // Create Docker client for testing (may be nil if not available) - dockerClient, _ := images.NewDockerClient() + // Create OCI client for testing + ociClient, err := images.NewOCIClient(cfg.DataDir + "/oci-cache") + if err != nil { + t.Fatalf("failed to create OCI client: %v", err) + } return &ApiService{ Config: cfg, - ImageManager: images.NewManager(cfg.DataDir, dockerClient), + ImageManager: images.NewManager(cfg.DataDir, ociClient), InstanceManager: instances.NewManager(cfg.DataDir), VolumeManager: volumes.NewManager(cfg.DataDir), } diff --git a/cmd/api/wire.go b/cmd/api/wire.go index e422c548..174e6fd5 100644 --- a/cmd/api/wire.go +++ b/cmd/api/wire.go @@ -32,7 +32,7 @@ func initializeApp() (*application, func(), error) { providers.ProvideLogger, providers.ProvideContext, providers.ProvideConfig, - providers.ProvideDockerClient, + providers.ProvideOCIClient, providers.ProvideImageManager, providers.ProvideInstanceManager, providers.ProvideVolumeManager, diff --git a/cmd/api/wire_gen.go b/cmd/api/wire_gen.go index 0ef19e26..aa06a69f 100644 --- a/cmd/api/wire_gen.go +++ b/cmd/api/wire_gen.go @@ -28,11 +28,11 @@ func initializeApp() (*application, func(), error) { logger := providers.ProvideLogger() context := providers.ProvideContext(logger) config := providers.ProvideConfig() - dockerClient, err := providers.ProvideDockerClient() + ociClient, err := providers.ProvideOCIClient(config) if err != nil { return nil, nil, err } - manager := providers.ProvideImageManager(config, dockerClient) + manager := providers.ProvideImageManager(config, ociClient) instancesManager := providers.ProvideInstanceManager(config) volumesManager := providers.ProvideVolumeManager(config) apiService := api.New(config, manager, instancesManager, volumesManager) diff --git a/go.mod b/go.mod index 0723f784..c964ca4a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/onkernel/hypeman go 1.25.4 require ( - github.com/docker/docker v28.5.2+incompatible + github.com/containers/image/v5 v5.36.2 github.com/getkin/kin-openapi v0.133.0 github.com/ghodss/yaml v1.0.0 github.com/go-chi/chi/v5 v5.2.3 @@ -12,59 +12,90 @@ require ( github.com/joho/godotenv v1.5.1 github.com/oapi-codegen/nethttp-middleware v1.1.2 github.com/oapi-codegen/runtime v1.1.2 + github.com/opencontainers/image-spec v1.1.1 + github.com/opencontainers/runtime-spec v1.2.1 + github.com/opencontainers/umoci v0.6.0 github.com/stretchr/testify v1.11.1 golang.org/x/sync v0.17.0 ) require ( - github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect + github.com/BurntSushi/toml v1.5.0 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/containerd/errdefs v0.3.0 // indirect - github.com/containerd/errdefs/pkg v0.3.0 // indirect - github.com/containerd/log v0.1.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/apex/log v1.9.0 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect + github.com/containers/ocicrypt v1.2.1 // indirect + github.com/containers/storage v1.59.1 // indirect + github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 // indirect + github.com/cyphar/filepath-securejoin v0.5.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/distribution v2.8.3+incompatible // indirect + github.com/docker/docker v28.5.2+incompatible // indirect + github.com/docker/docker-credential-helpers v0.9.3 // indirect + github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/felixge/httpsnoop v1.0.3 // indirect - github.com/go-logr/logr v1.4.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-jose/go-jose/v4 v4.0.5 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/go-containerregistry v0.20.3 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect + github.com/letsencrypt/boulder v0.0.0-20240620165639-de9c06129bec // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/go-archive v0.1.0 // indirect - github.com/moby/patternmatcher v0.6.0 // indirect - github.com/moby/sys/atomicwriter v0.1.0 // indirect - github.com/moby/sys/sequential v0.6.0 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mattn/go-sqlite3 v1.14.28 // indirect + github.com/miekg/pkcs11 v1.1.1 // indirect + github.com/moby/sys/capability v0.4.0 // indirect + github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect - github.com/moby/term v0.5.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect - github.com/morikuni/aec v1.0.0 // indirect github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/proglottis/gpgme v0.1.4 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect + github.com/rootless-containers/proto/go-proto v0.0.0-20230421021042-4cd87ebadd67 // indirect + github.com/secure-systems-lab/go-securesystemslib v0.9.0 // indirect + github.com/sigstore/fulcio v1.6.6 // indirect + github.com/sigstore/protobuf-specs v0.4.1 // indirect + github.com/sigstore/sigstore v1.9.5 // indirect + github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af // indirect + github.com/smallstep/pkcs7 v0.1.1 // indirect + github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect + github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect + github.com/vbatts/go-mtree v0.6.1-0.20250911112631-8307d76bc1b9 // indirect + github.com/vbatts/tar-split v0.12.1 // indirect + github.com/vbauerster/mpb/v8 v8.10.2 // indirect github.com/woodsbury/decimal128 v1.3.0 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect go.opentelemetry.io/otel v1.38.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 // indirect - go.opentelemetry.io/otel/metric v1.38.0 // indirect - go.opentelemetry.io/otel/sdk v1.21.0 // indirect - go.opentelemetry.io/otel/trace v1.38.0 // indirect - golang.org/x/net v0.28.0 // indirect - golang.org/x/sys v0.31.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect + go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect + golang.org/x/crypto v0.41.0 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/sys v0.37.0 // indirect + golang.org/x/term v0.34.0 // indirect + golang.org/x/text v0.28.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250313205543-e70fdf4c4cb4 // indirect + google.golang.org/grpc v1.72.2 // indirect + google.golang.org/protobuf v1.36.10 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 6b867a28..107c4ce9 100644 --- a/go.sum +++ b/go.sum @@ -1,41 +1,79 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= -github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= +github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= +github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= +github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= +github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= +github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/containerd/errdefs v0.3.0 h1:FSZgGOeK4yuT/+DnF07/Olde/q4KBoMsaamhXxIMDp4= -github.com/containerd/errdefs v0.3.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= -github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= -github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containers/image/v5 v5.36.2 h1:GcxYQyAHRF/pLqR4p4RpvKllnNL8mOBn0eZnqJbfTwk= +github.com/containers/image/v5 v5.36.2/go.mod h1:b4GMKH2z/5t6/09utbse2ZiLK/c72GuGLFdp7K69eA4= +github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA= +github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= +github.com/containers/ocicrypt v1.2.1 h1:0qIOTT9DoYwcKmxSt8QJt+VzMY18onl9jUXsxpVhSmM= +github.com/containers/ocicrypt v1.2.1/go.mod h1:aD0AAqfMp0MtwqWgHM1bUwe1anx0VazI108CRrSKINQ= +github.com/containers/storage v1.59.1 h1:11Zu68MXsEQGBBd+GadPrHPpWeqjKS8hJDGiAHgIqDs= +github.com/containers/storage v1.59.1/go.mod h1:KoAYHnAjP3/cTsRS+mmWZGkufSY2GACiKQ4V3ZLQnR0= +github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 h1:uX1JmpONuD549D73r6cgnxyUu18Zb7yHAy5AYU0Pm4Q= +github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw= +github.com/cyphar/filepath-securejoin v0.5.0 h1:hIAhkRBMQ8nIeuVwcAoymp7MY4oherZdAxD+m0u9zaw= +github.com/cyphar/filepath-securejoin v0.5.0/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/cli v28.3.2+incompatible h1:mOt9fcLE7zaACbxW1GeS65RI67wIJrTnqS3hP2huFsY= +github.com/docker/cli v28.3.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= +github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= +github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= -github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/getkin/kin-openapi v0.133.0 h1:pJdmNohVIJ97r4AUFtEXRXwESr8b0bD721u/Tz6k8PQ= github.com/getkin/kin-openapi v0.133.0/go.mod h1:boAciF6cXk5FhPqe/NQeBTeenbjqU4LhWBf09ILVvWE= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE= github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= +github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -44,53 +82,85 @@ github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1 github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= -github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= -github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= +github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-containerregistry v0.20.3 h1:oNx7IdTI936V8CQRveCjaxOiegWwvM7kqkbXTpyiovI= +github.com/google/go-containerregistry v0.20.3/go.mod h1:w00pIgBRDVUDFM6bq+Qx8lwNWK+cxgCuX1vd3PIBDNI= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmhodges/clock v1.2.0 h1:eq4kys+NI0PLngzaHEe7AmPT90XMGIEySD1JfV1PDIs= +github.com/jmhodges/clock v1.2.0/go.mod h1:qKjhA7x7u/lQpPB1XAqX1b1lCI/w3/fNuYpI/ZjLynI= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/letsencrypt/boulder v0.0.0-20240620165639-de9c06129bec h1:2tTW6cDth2TSgRbAhD7yjZzTQmcN25sDRPEeinR51yQ= +github.com/letsencrypt/boulder v0.0.0-20240620165639-de9c06129bec/go.mod h1:TmwEoGCwIti7BCeJ9hescZgRtatxRE+A72pCoPfmcfk= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= -github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= -github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= -github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= -github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= -github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= -github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= +github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= +github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/moby/sys/capability v0.4.0 h1:4D4mI6KlNtWMCM1Z/K0i7RV1FkX+DBDHKVJpCndZoHk= +github.com/moby/sys/capability v0.4.0/go.mod h1:4g9IK291rVkms3LKCDOoYlnV8xKwoDTpIrNEE35Wq0I= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= -github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= -github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/oapi-codegen/nethttp-middleware v1.1.2 h1:TQwEU3WM6ifc7ObBEtiJgbRPaCe513tvJpiMJjypVPA= github.com/oapi-codegen/nethttp-middleware v1.1.2/go.mod h1:5qzjxMSiI8HjLljiOEjvs4RdrWyMPKnExeFS2kr8om4= github.com/oapi-codegen/runtime v1.1.2 h1:P2+CubHq8fO4Q6fV1tqDBZHCwpVpvPg7oKiYzQgXIyI= @@ -99,74 +169,210 @@ github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//J github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037/go.mod h1:2bpvgLBZEtENV5scfDFEtB/5+1M4hkQhDQrccEJ/qGw= github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 h1:bQx3WeLcUWy+RletIKwUIt4x3t8n2SxavmoclizMb8c= github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU8lpJfSlR0xww= +github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/umoci v0.6.0 h1:Dsm4beJpglN5y2E2EUSZZcNey4Ml4+nKepvwLQwgIec= +github.com/opencontainers/umoci v0.6.0/go.mod h1:2DS3cxVN9pRJGYaCK5mnmmwVKV5vd9r6HIYAV0IvdbI= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/proglottis/gpgme v0.1.4 h1:3nE7YNA70o2aLjcg63tXMOhPD7bplfE5CBdV+hLAm2M= +github.com/proglottis/gpgme v0.1.4/go.mod h1:5LoXMgpE4bttgwwdv9bLs/vwqv3qV7F4glEEZ7mRKrM= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/rootless-containers/proto/go-proto v0.0.0-20230421021042-4cd87ebadd67 h1:58jvc5cZ+hGKidQ4Z37/+rj9eQxRRjOOsqNEwPSZXR4= +github.com/rootless-containers/proto/go-proto v0.0.0-20230421021042-4cd87ebadd67/go.mod h1:LLjEAc6zmycfeN7/1fxIphWQPjHpTt7ElqT7eVf8e4A= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/secure-systems-lab/go-securesystemslib v0.9.0 h1:rf1HIbL64nUpEIZnjLZ3mcNEL9NBPB0iuVjyxvq3LZc= +github.com/secure-systems-lab/go-securesystemslib v0.9.0/go.mod h1:DVHKMcZ+V4/woA/peqr+L0joiRXbPpQ042GgJckkFgw= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sigstore/fulcio v1.6.6 h1:XaMYX6TNT+8n7Npe8D94nyZ7/ERjEsNGFC+REdi/wzw= +github.com/sigstore/fulcio v1.6.6/go.mod h1:BhQ22lwaebDgIxVBEYOOqLRcN5+xOV+C9bh/GUXRhOk= +github.com/sigstore/protobuf-specs v0.4.1 h1:5SsMqZbdkcO/DNHudaxuCUEjj6x29tS2Xby1BxGU7Zc= +github.com/sigstore/protobuf-specs v0.4.1/go.mod h1:+gXR+38nIa2oEupqDdzg4qSBT0Os+sP7oYv6alWewWc= +github.com/sigstore/sigstore v1.9.5 h1:Wm1LT9yF4LhQdEMy5A2JeGRHTrAWGjT3ubE5JUSrGVU= +github.com/sigstore/sigstore v1.9.5/go.mod h1:VtxgvGqCmEZN9X2zhFSOkfXxvKUjpy8RpUW39oCtoII= +github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af h1:Sp5TG9f7K39yfB+If0vjp97vuT74F72r8hfRpP8jLU0= +github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/smallstep/pkcs7 v0.1.1 h1:x+rPdt2W088V9Vkjho4KtoggyktZJlMduZAtRHm68LU= +github.com/smallstep/pkcs7 v0.1.1/go.mod h1:dL6j5AIz9GHjVEBTXtW+QliALcgM19RtXaTeyxI+AfA= +github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= +github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 h1:pnnLyeX7o/5aX8qUQ69P/mLojDqwda8hFOCBTmP/6hw= +github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6/go.mod h1:39R/xuhNgVhi+K0/zst4TLrJrVmbm6LVgl4A0+ZFS5M= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C1wj2THlRK+oAhjeS/TRQwMfkIuet3w0= +github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs= +github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= +github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= +github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc= +github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= +github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= +github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/vbatts/go-mtree v0.6.1-0.20250911112631-8307d76bc1b9 h1:R6l9BtUe83abUGu1YKGkfa17wMMFLt6mhHVQ8MxpfRE= +github.com/vbatts/go-mtree v0.6.1-0.20250911112631-8307d76bc1b9/go.mod h1:W7bcG9PCn6lFY+ljGlZxx9DONkxL3v8a7HyN+PrSrjA= +github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo= +github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= +github.com/vbauerster/mpb/v8 v8.10.2 h1:2uBykSHAYHekE11YvJhKxYmLATKHAGorZwFlyNw4hHM= +github.com/vbauerster/mpb/v8 v8.10.2/go.mod h1:+Ja4P92E3/CorSZgfDtK46D7AVbDqmBQRTmyTqPElo0= github.com/woodsbury/decimal128 v1.3.0 h1:8pffMNWIlC0O5vbyHWFZAt5yWvWcrHA+3ovIIjVWss0= github.com/woodsbury/decimal128 v1.3.0/go.mod h1:C5UTmyTjW3JftjUFzOVhC20BEQa2a4ZKOB5I6Zjb+ds= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= -go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= -go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= -go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.67.0 h1:IdH9y6PF5MPSdAntIcpjQ+tXO41pcQsfZV2RxtQgVcw= -google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= +google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250313205543-e70fdf4c4cb4 h1:iK2jbkWL86DXjEx0qiHcRE9dE4/Ahua5k6V8OWFb//c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250313205543-e70fdf4c4cb4/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= +google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= +google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= -gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/lib/images/disk.go b/lib/images/disk.go new file mode 100644 index 00000000..e3a5e34f --- /dev/null +++ b/lib/images/disk.go @@ -0,0 +1,72 @@ +package images + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" +) + +// convertToExt4 converts a rootfs directory to an ext4 disk image using mkfs.ext4 +func convertToExt4(rootfsDir, diskPath string) (int64, error) { + // Calculate size of rootfs directory + sizeBytes, err := dirSize(rootfsDir) + if err != nil { + return 0, fmt.Errorf("calculate dir size: %w", err) + } + + // Add 20% overhead for filesystem metadata, minimum 1GB + diskSizeBytes := sizeBytes + (sizeBytes / 5) + const minSize = 1024 * 1024 * 1024 // 1GB + if diskSizeBytes < minSize { + diskSizeBytes = minSize + } + + // Ensure parent directory exists + if err := os.MkdirAll(filepath.Dir(diskPath), 0755); err != nil { + return 0, fmt.Errorf("create disk parent dir: %w", err) + } + + // Create sparse file + f, err := os.Create(diskPath) + if err != nil { + return 0, fmt.Errorf("create disk file: %w", err) + } + if err := f.Truncate(diskSizeBytes); err != nil { + f.Close() + return 0, fmt.Errorf("truncate disk file: %w", err) + } + f.Close() + + // Format as ext4 with rootfs contents using mkfs.ext4 + // This works without root when creating filesystem in a regular file + cmd := exec.Command("mkfs.ext4", "-d", rootfsDir, "-F", diskPath) + output, err := cmd.CombinedOutput() + if err != nil { + return 0, fmt.Errorf("mkfs.ext4 failed: %w, output: %s", err, output) + } + + // Get actual disk size + stat, err := os.Stat(diskPath) + if err != nil { + return 0, fmt.Errorf("stat disk: %w", err) + } + + return stat.Size(), nil +} + +// dirSize calculates the total size of a directory +func dirSize(path string) (int64, error) { + var size int64 + err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + size += info.Size() + } + return nil + }) + return size, err +} + diff --git a/lib/images/docker.go b/lib/images/docker.go deleted file mode 100644 index a91c66ab..00000000 --- a/lib/images/docker.go +++ /dev/null @@ -1,159 +0,0 @@ -package images - -import ( - "archive/tar" - "context" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/image" - "github.com/docker/docker/client" -) - -// DockerClient wraps Docker API operations -type DockerClient struct { - cli *client.Client -} - -// NewDockerClient creates a new Docker client using environment variables -func NewDockerClient() (*DockerClient, error) { - cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - if err != nil { - return nil, fmt.Errorf("create docker client: %w", err) - } - return &DockerClient{cli: cli}, nil -} - -// Close closes the Docker client -func (d *DockerClient) Close() error { - return d.cli.Close() -} - -// pullAndExport pulls an OCI image and exports its rootfs to a directory -func (d *DockerClient) pullAndExport(ctx context.Context, imageRef, exportDir string) (*containerMetadata, error) { - // Pull the image - pullReader, err := d.cli.ImagePull(ctx, imageRef, image.PullOptions{}) - if err != nil { - return nil, fmt.Errorf("pull image: %w", err) - } - // Consume the pull output (required for pull to complete) - io.Copy(io.Discard, pullReader) - pullReader.Close() - - // Inspect image to get metadata - inspect, _, err := d.cli.ImageInspectWithRaw(ctx, imageRef) - if err != nil { - return nil, fmt.Errorf("inspect image: %w", err) - } - - // Extract metadata from inspect response - meta := &containerMetadata{ - Entrypoint: inspect.Config.Entrypoint, - Cmd: inspect.Config.Cmd, - Env: make(map[string]string), - WorkingDir: inspect.Config.WorkingDir, - } - - // Parse environment variables - for _, env := range inspect.Config.Env { - for i := 0; i < len(env); i++ { - if env[i] == '=' { - key := env[:i] - val := env[i+1:] - meta.Env[key] = val - break - } - } - } - - // Create a temporary container from the image - // Creating a container does not run it, - // it does not execute any code inside the container. - containerResp, err := d.cli.ContainerCreate(ctx, &container.Config{ - Image: imageRef, - }, nil, nil, nil, "") - if err != nil { - return nil, fmt.Errorf("create container: %w", err) - } - - // Ensure container is removed even if export fails - defer func() { - d.cli.ContainerRemove(ctx, containerResp.ID, container.RemoveOptions{}) - }() - - // Export container filesystem as tar stream - exportReader, err := d.cli.ContainerExport(ctx, containerResp.ID) - if err != nil { - return nil, fmt.Errorf("export container: %w", err) - } - defer exportReader.Close() - - // Extract tar to exportDir - if err := extractTar(exportReader, exportDir); err != nil { - return nil, fmt.Errorf("extract tar: %w", err) - } - - return meta, nil -} - -// containerMetadata holds extracted container metadata -type containerMetadata struct { - Entrypoint []string - Cmd []string - Env map[string]string - WorkingDir string -} - -// extractTar extracts a tar stream to a target directory -// Ignores ownership/permission errors (matches POC behavior: docker export | tar -C dir -xf -) -func extractTar(reader io.Reader, targetDir string) error { - tarReader := tar.NewReader(reader) - - for { - header, err := tarReader.Next() - if err == io.EOF { - break - } - if err != nil { - return fmt.Errorf("read tar header: %w", err) - } - - target := filepath.Join(targetDir, header.Name) - - switch header.Typeflag { - case tar.TypeDir: - // Create directory, ignore chmod errors (best effort) - os.MkdirAll(target, 0755) - - case tar.TypeReg: - // Create parent directory - os.MkdirAll(filepath.Dir(target), 0755) - - // Create file, use default permissions if header.Mode fails - outFile, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - return fmt.Errorf("create file %s: %w", target, err) - } - if _, err := io.Copy(outFile, tarReader); err != nil { - outFile.Close() - return fmt.Errorf("write file %s: %w", target, err) - } - outFile.Close() - - case tar.TypeSymlink: - os.MkdirAll(filepath.Dir(target), 0755) - os.Remove(target) // Remove if exists - os.Symlink(header.Linkname, target) // Ignore errors - - case tar.TypeLink: - linkTarget := filepath.Join(targetDir, header.Linkname) - os.Remove(target) - os.Link(linkTarget, target) // Ignore errors - } - } - - return nil -} diff --git a/lib/images/manager.go b/lib/images/manager.go index ba3faf3b..a6c89470 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -2,10 +2,8 @@ package images import ( "context" - "crypto/sha256" "fmt" "os" - "os/exec" "path/filepath" "regexp" "strings" @@ -23,15 +21,15 @@ type Manager interface { } type manager struct { - dataDir string - dockerClient *DockerClient + dataDir string + ociClient *OCIClient } -// NewManager creates a new image manager with Docker client -func NewManager(dataDir string, dockerClient *DockerClient) Manager { +// NewManager creates a new image manager with OCI client +func NewManager(dataDir string, ociClient *OCIClient) Manager { return &manager{ - dataDir: dataDir, - dockerClient: dockerClient, + dataDir: dataDir, + ociClient: ociClient, } } @@ -66,7 +64,7 @@ func (m *manager) CreateImage(ctx context.Context, req oapi.CreateImageRequest) tempDir := filepath.Join(os.TempDir(), fmt.Sprintf("hypeman-image-%s-%d", *imageID, time.Now().Unix())) defer os.RemoveAll(tempDir) // cleanup temp dir - containerMeta, err := m.dockerClient.pullAndExport(ctx, req.Name, tempDir) + containerMeta, err := m.ociClient.pullAndExport(ctx, req.Name, tempDir) if err != nil { return nil, fmt.Errorf("pull and export: %w", err) } @@ -128,75 +126,4 @@ func generateImageID(imageName string) string { return "img-" + sanitized } -// convertToExt4 converts a rootfs directory to an ext4 disk image -func convertToExt4(rootfsDir, diskPath string) (int64, error) { - // Calculate size of rootfs directory (rounded up to nearest GB, minimum 1GB) - sizeBytes, err := dirSize(rootfsDir) - if err != nil { - return 0, fmt.Errorf("calculate dir size: %w", err) - } - - // Add 20% overhead for filesystem metadata, minimum 1GB - diskSizeBytes := sizeBytes + (sizeBytes / 5) - const minSize = 1024 * 1024 * 1024 // 1GB - if diskSizeBytes < minSize { - diskSizeBytes = minSize - } - - // Ensure parent directory exists - if err := os.MkdirAll(filepath.Dir(diskPath), 0755); err != nil { - return 0, fmt.Errorf("create disk parent dir: %w", err) - } - - // Create sparse file - f, err := os.Create(diskPath) - if err != nil { - return 0, fmt.Errorf("create disk file: %w", err) - } - if err := f.Truncate(diskSizeBytes); err != nil { - f.Close() - return 0, fmt.Errorf("truncate disk file: %w", err) - } - f.Close() - - // Format as ext4 with rootfs contents - cmd := exec.Command("mkfs.ext4", "-d", rootfsDir, "-F", diskPath) - output, err := cmd.CombinedOutput() - if err != nil { - return 0, fmt.Errorf("mkfs.ext4 failed: %w, output: %s", err, output) - } - - // Get actual disk size - stat, err := os.Stat(diskPath) - if err != nil { - return 0, fmt.Errorf("stat disk: %w", err) - } - - return stat.Size(), nil -} - -// dirSize calculates the total size of a directory -func dirSize(path string) (int64, error) { - var size int64 - err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - size += info.Size() - } - return nil - }) - return size, err -} - -// checksumFile computes sha256 of a file -func checksumFile(path string) (string, error) { - data, err := os.ReadFile(path) - if err != nil { - return "", err - } - hash := sha256.Sum256(data) - return fmt.Sprintf("%x", hash), nil -} diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go index d52e9e9f..fa9a1b45 100644 --- a/lib/images/manager_test.go +++ b/lib/images/manager_test.go @@ -11,10 +11,11 @@ import ( ) func TestCreateImage(t *testing.T) { - dockerClient := requireDocker(t) + ociClient, err := NewOCIClient(t.TempDir()) + require.NoError(t, err) dataDir := t.TempDir() - mgr := NewManager(dataDir, dockerClient) + mgr := NewManager(dataDir, ociClient) ctx := context.Background() req := oapi.CreateImageRequest{ @@ -41,10 +42,11 @@ func TestCreateImage(t *testing.T) { } func TestCreateImageWithCustomID(t *testing.T) { - dockerClient := requireDocker(t) + ociClient, err := NewOCIClient(t.TempDir()) + require.NoError(t, err) dataDir := t.TempDir() - mgr := NewManager(dataDir, dockerClient) + mgr := NewManager(dataDir, ociClient) ctx := context.Background() customID := "my-custom-alpine" @@ -60,10 +62,11 @@ func TestCreateImageWithCustomID(t *testing.T) { } func TestCreateImageDuplicate(t *testing.T) { - dockerClient := requireDocker(t) + ociClient, err := NewOCIClient(t.TempDir()) + require.NoError(t, err) dataDir := t.TempDir() - mgr := NewManager(dataDir, dockerClient) + mgr := NewManager(dataDir, ociClient) ctx := context.Background() req := oapi.CreateImageRequest{ @@ -71,7 +74,7 @@ func TestCreateImageDuplicate(t *testing.T) { } // Create first image - _, err := mgr.CreateImage(ctx, req) + _, err = mgr.CreateImage(ctx, req) require.NoError(t, err) // Try to create duplicate @@ -80,10 +83,11 @@ func TestCreateImageDuplicate(t *testing.T) { } func TestListImages(t *testing.T) { - dockerClient := requireDocker(t) + ociClient, err := NewOCIClient(t.TempDir()) + require.NoError(t, err) dataDir := t.TempDir() - mgr := NewManager(dataDir, dockerClient) + mgr := NewManager(dataDir, ociClient) ctx := context.Background() @@ -107,10 +111,11 @@ func TestListImages(t *testing.T) { } func TestGetImage(t *testing.T) { - dockerClient := requireDocker(t) + ociClient, err := NewOCIClient(t.TempDir()) + require.NoError(t, err) dataDir := t.TempDir() - mgr := NewManager(dataDir, dockerClient) + mgr := NewManager(dataDir, ociClient) ctx := context.Background() req := oapi.CreateImageRequest{ @@ -130,26 +135,25 @@ func TestGetImage(t *testing.T) { } func TestGetImageNotFound(t *testing.T) { - dockerClient, _ := NewDockerClient() - if dockerClient != nil { - defer dockerClient.Close() - } + ociClient, err := NewOCIClient(t.TempDir()) + require.NoError(t, err) dataDir := t.TempDir() - mgr := NewManager(dataDir, dockerClient) + mgr := NewManager(dataDir, ociClient) ctx := context.Background() // Try to get non-existent image - _, err := mgr.GetImage(ctx, "nonexistent") + _, err = mgr.GetImage(ctx, "nonexistent") require.ErrorIs(t, err, ErrNotFound) } func TestDeleteImage(t *testing.T) { - dockerClient := requireDocker(t) + ociClient, err := NewOCIClient(t.TempDir()) + require.NoError(t, err) dataDir := t.TempDir() - mgr := NewManager(dataDir, dockerClient) + mgr := NewManager(dataDir, ociClient) ctx := context.Background() req := oapi.CreateImageRequest{ @@ -174,18 +178,16 @@ func TestDeleteImage(t *testing.T) { } func TestDeleteImageNotFound(t *testing.T) { - dockerClient, _ := NewDockerClient() - if dockerClient != nil { - defer dockerClient.Close() - } + ociClient, err := NewOCIClient(t.TempDir()) + require.NoError(t, err) dataDir := t.TempDir() - mgr := NewManager(dataDir, dockerClient) + mgr := NewManager(dataDir, ociClient) ctx := context.Background() // Try to delete non-existent image - err := mgr.DeleteImage(ctx, "nonexistent") + err = mgr.DeleteImage(ctx, "nonexistent") require.ErrorIs(t, err, ErrNotFound) } @@ -209,23 +211,4 @@ func TestGenerateImageID(t *testing.T) { } } -// requireDocker fails the test if Docker is not available or accessible -// Returns a DockerClient for use in tests -func requireDocker(t *testing.T) *DockerClient { - // Try to connect to Docker to verify we have permission - client, err := NewDockerClient() - if err != nil { - t.Fatalf("cannot connect to docker: %v", err) - } - - // Verify we can actually use Docker by pinging it - ctx := context.Background() - _, err = client.cli.Ping(ctx) - if err != nil { - client.Close() - t.Fatalf("docker not available: %v", err) - } - - return client -} diff --git a/lib/images/oci.go b/lib/images/oci.go new file mode 100644 index 00000000..2286ceed --- /dev/null +++ b/lib/images/oci.go @@ -0,0 +1,240 @@ +package images + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/containers/image/v5/copy" + "github.com/containers/image/v5/docker" + "github.com/containers/image/v5/oci/layout" + "github.com/containers/image/v5/signature" + "github.com/opencontainers/image-spec/specs-go/v1" + rspec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/umoci/oci/cas/dir" + "github.com/opencontainers/umoci/oci/casext" + "github.com/opencontainers/umoci/oci/layer" +) + +// OCIClient handles OCI image operations without requiring Docker daemon +type OCIClient struct { + cacheDir string +} + +// NewOCIClient creates a new OCI client +func NewOCIClient(cacheDir string) (*OCIClient, error) { + if err := os.MkdirAll(cacheDir, 0755); err != nil { + return nil, fmt.Errorf("create cache dir: %w", err) + } + return &OCIClient{cacheDir: cacheDir}, nil +} + +// Close closes the OCI client (no-op for now) +func (c *OCIClient) Close() error { + return nil +} + +// pullAndExport pulls an OCI image and exports its rootfs to a directory +func (c *OCIClient) pullAndExport(ctx context.Context, imageRef, exportDir string) (*containerMetadata, error) { + // Create temporary OCI layout directory + ociLayoutDir := filepath.Join(c.cacheDir, fmt.Sprintf("oci-layout-%d", os.Getpid())) + if err := os.MkdirAll(ociLayoutDir, 0755); err != nil { + return nil, fmt.Errorf("create oci layout dir: %w", err) + } + defer os.RemoveAll(ociLayoutDir) + + // Pull image to OCI layout + if err := c.pullToOCILayout(ctx, imageRef, ociLayoutDir); err != nil { + return nil, fmt.Errorf("pull to oci layout: %w", err) + } + + // Extract metadata from OCI config + meta, err := c.extractOCIMetadata(ociLayoutDir) + if err != nil { + return nil, fmt.Errorf("extract metadata: %w", err) + } + + // Unpack layers to export directory + if err := c.unpackLayers(ctx, ociLayoutDir, exportDir); err != nil { + return nil, fmt.Errorf("unpack layers: %w", err) + } + + return meta, nil +} + +// pullToOCILayout pulls an image from a registry to an OCI layout directory +func (c *OCIClient) pullToOCILayout(ctx context.Context, imageRef, ociLayoutDir string) error { + // Parse source reference (docker://...) + srcRef, err := docker.ParseReference("//" + imageRef) + if err != nil { + return fmt.Errorf("parse image reference: %w", err) + } + + // Create destination reference (OCI layout) + destRef, err := layout.ParseReference(ociLayoutDir + ":latest") + if err != nil { + return fmt.Errorf("parse oci layout reference: %w", err) + } + + // Create policy context (allow all) + policyContext, err := signature.NewPolicyContext(&signature.Policy{ + Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}, + }) + if err != nil { + return fmt.Errorf("create policy context: %w", err) + } + defer policyContext.Destroy() + + // Pull image + _, err = copy.Image(ctx, policyContext, destRef, srcRef, ©.Options{ + ReportWriter: os.Stdout, + }) + if err != nil { + return fmt.Errorf("copy image: %w", err) + } + + return nil +} + +// extractOCIMetadata reads metadata from OCI layout config.json +func (c *OCIClient) extractOCIMetadata(ociLayoutDir string) (*containerMetadata, error) { + // Open the OCI layout + casEngine, err := dir.Open(ociLayoutDir) + if err != nil { + return nil, fmt.Errorf("open oci layout: %w", err) + } + defer casEngine.Close() + + engine := casext.NewEngine(casEngine) + + // Get image reference (we tagged it as "latest") + descriptorPaths, err := engine.ResolveReference(context.Background(), "latest") + if err != nil { + return nil, fmt.Errorf("resolve reference: %w", err) + } + + if len(descriptorPaths) == 0 { + return nil, fmt.Errorf("no image found in oci layout") + } + + // Get the manifest + manifestBlob, err := engine.FromDescriptor(context.Background(), descriptorPaths[0].Descriptor()) + if err != nil { + return nil, fmt.Errorf("get manifest: %w", err) + } + + // casext automatically parses manifests, so Data is already a v1.Manifest + manifest, ok := manifestBlob.Data.(v1.Manifest) + if !ok { + return nil, fmt.Errorf("manifest data is not v1.Manifest (got %T)", manifestBlob.Data) + } + + // Get the config blob + configBlob, err := engine.FromDescriptor(context.Background(), manifest.Config) + if err != nil { + return nil, fmt.Errorf("get config: %w", err) + } + + // casext automatically parses config, so Data is already a v1.Image + config, ok := configBlob.Data.(v1.Image) + if !ok { + return nil, fmt.Errorf("config data is not v1.Image (got %T)", configBlob.Data) + } + + // Extract metadata + meta := &containerMetadata{ + Entrypoint: config.Config.Entrypoint, + Cmd: config.Config.Cmd, + Env: make(map[string]string), + WorkingDir: config.Config.WorkingDir, + } + + // Parse environment variables + for _, env := range config.Config.Env { + for i := 0; i < len(env); i++ { + if env[i] == '=' { + key := env[:i] + val := env[i+1:] + meta.Env[key] = val + break + } + } + } + + return meta, nil +} + +// unpackLayers unpacks all OCI layers to a target directory using umoci +func (c *OCIClient) unpackLayers(ctx context.Context, ociLayoutDir, targetDir string) error { + // Open OCI layout + casEngine, err := dir.Open(ociLayoutDir) + if err != nil { + return fmt.Errorf("open oci layout: %w", err) + } + defer casEngine.Close() + + engine := casext.NewEngine(casEngine) + + // Get the manifest descriptor for "latest" tag + descriptorPaths, err := engine.ResolveReference(context.Background(), "latest") + if err != nil { + return fmt.Errorf("resolve reference: %w", err) + } + + if len(descriptorPaths) == 0 { + return fmt.Errorf("no image found") + } + + // Get the manifest blob + manifestBlob, err := engine.FromDescriptor(context.Background(), descriptorPaths[0].Descriptor()) + if err != nil { + return fmt.Errorf("get manifest: %w", err) + } + + // casext automatically parses manifests + manifest, ok := manifestBlob.Data.(v1.Manifest) + if !ok { + return fmt.Errorf("manifest data is not v1.Manifest (got %T)", manifestBlob.Data) + } + + // Pre-create target directory (umoci needs it to exist) + if err := os.MkdirAll(targetDir, 0755); err != nil { + return fmt.Errorf("create target dir: %w", err) + } + + // Unpack layers using umoci's layer package with rootless mode + // Map container UIDs to current user's UID (identity mapping) + uid := uint32(os.Getuid()) + gid := uint32(os.Getgid()) + + unpackOpts := &layer.UnpackOptions{ + OnDiskFormat: layer.DirRootfs{ + MapOptions: layer.MapOptions{ + Rootless: true, // Don't fail on chown errors + UIDMappings: []rspec.LinuxIDMapping{ + {HostID: uid, ContainerID: 0, Size: 1}, // Map container root to current user + }, + GIDMappings: []rspec.LinuxIDMapping{ + {HostID: gid, ContainerID: 0, Size: 1}, // Map container root group to current user group + }, + }, + }, + } + + err = layer.UnpackRootfs(context.Background(), casEngine, targetDir, manifest, unpackOpts) + if err != nil { + return fmt.Errorf("unpack rootfs: %w", err) + } + + return nil +} + +// containerMetadata holds extracted container metadata +type containerMetadata struct { + Entrypoint []string + Cmd []string + Env map[string]string + WorkingDir string +} + diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 3d7f2cc7..1fcefb41 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -29,14 +29,16 @@ func ProvideConfig() *config.Config { return config.Load() } -// ProvideDockerClient provides a Docker client -func ProvideDockerClient() (*images.DockerClient, error) { - return images.NewDockerClient() +// ProvideOCIClient provides an OCI client +func ProvideOCIClient(cfg *config.Config) (*images.OCIClient, error) { + // Use a cache directory under dataDir for OCI layouts + cacheDir := cfg.DataDir + "/oci-cache" + return images.NewOCIClient(cacheDir) } // ProvideImageManager provides the image manager -func ProvideImageManager(cfg *config.Config, dockerClient *images.DockerClient) images.Manager { - return images.NewManager(cfg.DataDir, dockerClient) +func ProvideImageManager(cfg *config.Config, ociClient *images.OCIClient) images.Manager { + return images.NewManager(cfg.DataDir, ociClient) } // ProvideInstanceManager provides the instance manager From 3cf8089368736ce7d1acbbc01165f255247e73cb Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 7 Nov 2025 10:42:37 -0500 Subject: [PATCH 07/37] Update docs --- README.md | 14 ++------- lib/images/README.md | 69 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 11 deletions(-) create mode 100644 lib/images/README.md diff --git a/README.md b/README.md index 618f9d22..08314f28 100644 --- a/README.md +++ b/README.md @@ -8,21 +8,13 @@ Run containerized workloads in VMs, powered by [Cloud Hypervisor](https://github ### Prerequisites -**Cloud Hypervisor** - [Installation guide](https://www.cloudhypervisor.org/docs/prologue/quick-start/#use-pre-built-binaries) -```bash -cloud-hypervisor --version # Verify -ch-remote --version -``` +**Go 1.25.4+**, **Cloud Hypervisor**, **KVM**, **mkfs.ext4** -**Docker** - [Installation guide](https://docs.docker.com/engine/install/) ```bash -docker --version # Verify -# Add your user to docker group for non-root access: -sudo usermod -aG docker $USER +cloud-hypervisor --version +mkfs.ext4 -V ``` -**Go 1.25.4+** and **KVM** - ### Configuration ```bash diff --git a/lib/images/README.md b/lib/images/README.md new file mode 100644 index 00000000..d01c18b8 --- /dev/null +++ b/lib/images/README.md @@ -0,0 +1,69 @@ +# Image Manager + +Converts OCI container images into bootable ext4 disk images for Cloud Hypervisor VMs. + +## Architecture + +``` +OCI Registry → containers/image → OCI Layout → umoci → rootfs/ → mkfs.ext4 → disk.ext4 +``` + +## Design Decisions + +### Why containers/image? (oci.go) + +**What:** Pull OCI images from any registry (Docker Hub, ghcr.io, etc.) + +**Why:** +- Standard library used by Podman, Skopeo, Buildah +- Works directly with registries (no daemon required) +- Supports all registry authentication methods + +**Alternative:** Docker API - requires Docker daemon running + +### Why umoci? (oci.go) + +**What:** Unpack OCI image layers in userspace + +**Why:** +- Purpose-built for rootless OCI manipulation (official OpenContainers project) +- Handles OCI layer semantics (whiteouts, layer ordering) correctly +- Designed to work without root privileges + +**Alternative:** With Docker API, the daemon (running as root) mounts image layers using overlayfs, then exports the merged filesystem. Users get the result without needing root themselves but it still has the dependency on Docker and does actually mount the overlays to get the merged filesystem. With umoci, layers are merged in userspace by extracting each tar layer sequentially and applying changes (including whiteouts for deletions). No kernel mount needed, fully rootless. Umoci was chosen because it's purpose-built for this use case and embeddable with the go program. + +### Why mkfs.ext4? (disk.go) + +**What:** Shell command to create ext4 filesystem + +**Why:** +- Battle-tested, fast, reliable +- Works without root when creating filesystem in a regular file (not block device) +- `-d` flag copies directory contents into filesystem + +**Alternative tried:** go-diskfs pure Go ext4, got too tricky but could revisit this + +**Tradeoff:** Shell command vs pure Go, but mkfs.ext4 is universally available and robust + +## Filesystem Persistence (storage.go) + +**Metadata:** JSON files with atomic writes (tmp + rename) +``` +/var/lib/hypeman/images/{id}/ + rootfs.ext4 + metadata.json +``` + +**Why filesystem vs database?** +- Disk images must be on filesystem anyway +- No sync issues between DB and actual artifacts +- Simple recovery (scan directory to rebuild state) + +## Build Tags + +Requires `-tags containers_image_openpgp` to avoid C dependency on gpgme. + +## Testing + +Integration tests pull alpine:latest (~7MB) and verify full pipeline. No special permissions required. + From 2dcfde54e505c1e8e2ec28152ef96ee629396616 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 7 Nov 2025 10:49:00 -0500 Subject: [PATCH 08/37] Update README --- .air.toml | 2 +- README.md | 11 +++++++++++ lib/images/README.md | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.air.toml b/.air.toml index d30c8963..05d74ce2 100644 --- a/.air.toml +++ b/.air.toml @@ -5,7 +5,7 @@ tmp_dir = "tmp" [build] args_bin = [] bin = "./tmp/main" - cmd = "go build -o ./tmp/main ./cmd/api" + cmd = "go build -tags containers_image_openpgp -o ./tmp/main ./cmd/api" delay = 1000 exclude_dir = ["assets", "tmp", "vendor", "testdata", "bin", "scripts", "data", "kernel"] exclude_file = [] diff --git a/README.md b/README.md index 08314f28..626ec487 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,22 @@ mkfs.ext4 -V ### Configuration +#### Environment variables + ```bash cp .env.example .env # Edit .env and set JWT_SECRET ``` +#### Data directory + +Hypeman stores data in a configurable directory. Configure permissions for this directory. + +```bash +sudo mkdir /var/lib/hypeman +sudo chown $USER:$USER /var/lib/hypeman +``` + ### Build ```bash diff --git a/lib/images/README.md b/lib/images/README.md index d01c18b8..65ebbb89 100644 --- a/lib/images/README.md +++ b/lib/images/README.md @@ -61,7 +61,7 @@ OCI Registry → containers/image → OCI Layout → umoci → rootfs/ → mkfs. ## Build Tags -Requires `-tags containers_image_openpgp` to avoid C dependency on gpgme. +Requires `-tags containers_image_openpgp` to avoid C dependency on gpgme. This is a build-time option of the containers/image project to select between gpgme C library with go bindings or the pure Go OpenPGP implementation (slightly slower but doesn't need external system dependency). ## Testing From 9c79b184745782340cd717880bd7493efef231c7 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 7 Nov 2025 10:59:21 -0500 Subject: [PATCH 09/37] Move OCI cache dir --- cmd/api/api/api_test.go | 2 +- lib/providers/providers.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/api/api/api_test.go b/cmd/api/api/api_test.go index 6328b61b..091c9b08 100644 --- a/cmd/api/api/api_test.go +++ b/cmd/api/api/api_test.go @@ -17,7 +17,7 @@ func newTestService(t *testing.T) *ApiService { } // Create OCI client for testing - ociClient, err := images.NewOCIClient(cfg.DataDir + "/oci-cache") + ociClient, err := images.NewOCIClient(cfg.DataDir + "/system/oci-cache") if err != nil { t.Fatalf("failed to create OCI client: %v", err) } diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 1fcefb41..888696df 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -32,7 +32,7 @@ func ProvideConfig() *config.Config { // ProvideOCIClient provides an OCI client func ProvideOCIClient(cfg *config.Config) (*images.OCIClient, error) { // Use a cache directory under dataDir for OCI layouts - cacheDir := cfg.DataDir + "/oci-cache" + cacheDir := cfg.DataDir + "/system/oci-cache" return images.NewOCIClient(cacheDir) } From db95987d7812f03ed59de34ad4b3f6021e1105b5 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 7 Nov 2025 11:22:05 -0500 Subject: [PATCH 10/37] Tweak disk settings --- lib/images/README.md | 7 ++++++- lib/images/disk.go | 11 +++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/images/README.md b/lib/images/README.md index 65ebbb89..daf63c40 100644 --- a/lib/images/README.md +++ b/lib/images/README.md @@ -41,7 +41,12 @@ OCI Registry → containers/image → OCI Layout → umoci → rootfs/ → mkfs. - Works without root when creating filesystem in a regular file (not block device) - `-d` flag copies directory contents into filesystem -**Alternative tried:** go-diskfs pure Go ext4, got too tricky but could revisit this +**Options:** +- `-b 4096` - 4KB blocks (standard, matches VM page size) +- `-O ^has_journal` - No journal (disks mounted read-only in VMs, saves ~32MB) +- Minimum 10MB size covers ext4 metadata (~5MB for superblock, inodes, bitmaps) + +**Alternative tried:** go-diskfs pure Go ext4 - has bugs **Tradeoff:** Shell command vs pure Go, but mkfs.ext4 is universally available and robust diff --git a/lib/images/disk.go b/lib/images/disk.go index e3a5e34f..09445b95 100644 --- a/lib/images/disk.go +++ b/lib/images/disk.go @@ -15,9 +15,9 @@ func convertToExt4(rootfsDir, diskPath string) (int64, error) { return 0, fmt.Errorf("calculate dir size: %w", err) } - // Add 20% overhead for filesystem metadata, minimum 1GB + // Add 20% overhead for filesystem metadata, minimum 10MB diskSizeBytes := sizeBytes + (sizeBytes / 5) - const minSize = 1024 * 1024 * 1024 // 1GB + const minSize = 10 * 1024 * 1024 // 10MB if diskSizeBytes < minSize { diskSizeBytes = minSize } @@ -39,8 +39,11 @@ func convertToExt4(rootfsDir, diskPath string) (int64, error) { f.Close() // Format as ext4 with rootfs contents using mkfs.ext4 - // This works without root when creating filesystem in a regular file - cmd := exec.Command("mkfs.ext4", "-d", rootfsDir, "-F", diskPath) + // -b 4096: 4KB blocks (standard, matches VM page size) + // -O ^has_journal: Disable journal (not needed for read-only VM mounts) + // -d: Copy directory contents into filesystem + // -F: Force creation (file not block device) + cmd := exec.Command("mkfs.ext4", "-b", "4096", "-O", "^has_journal", "-d", rootfsDir, "-F", diskPath) output, err := cmd.CombinedOutput() if err != nil { return 0, fmt.Errorf("mkfs.ext4 failed: %w, output: %s", err, output) From 4e13b7c938ead3eeaf41f797ca7885a95a8a8d2f Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 7 Nov 2025 12:59:42 -0500 Subject: [PATCH 11/37] Async API --- cmd/api/api/api_test.go | 2 +- cmd/api/api/images.go | 29 ++- cmd/api/api/images_test.go | 119 ++++++++++++ cmd/api/config/config.go | 40 ++-- lib/images/manager.go | 182 +++++++++++++++--- lib/images/manager_test.go | 81 ++++++-- lib/images/oci.go | 47 ++++- lib/images/progress.go | 225 ++++++++++++++++++++++ lib/images/queue.go | 111 +++++++++++ lib/images/storage.go | 39 ++-- lib/oapi/oapi.go | 381 +++++++++++++++++++++++++++++++------ lib/providers/providers.go | 2 +- openapi.yaml | 62 +++++- 13 files changed, 1181 insertions(+), 139 deletions(-) create mode 100644 lib/images/progress.go create mode 100644 lib/images/queue.go diff --git a/cmd/api/api/api_test.go b/cmd/api/api/api_test.go index 091c9b08..c1ac23de 100644 --- a/cmd/api/api/api_test.go +++ b/cmd/api/api/api_test.go @@ -24,7 +24,7 @@ func newTestService(t *testing.T) *ApiService { return &ApiService{ Config: cfg, - ImageManager: images.NewManager(cfg.DataDir, ociClient), + ImageManager: images.NewManager(cfg.DataDir, ociClient, 1), InstanceManager: instances.NewManager(cfg.DataDir), VolumeManager: volumes.NewManager(cfg.DataDir), } diff --git a/cmd/api/api/images.go b/cmd/api/api/images.go index 470cee31..ae265d80 100644 --- a/cmd/api/api/images.go +++ b/cmd/api/api/images.go @@ -44,7 +44,7 @@ func (s *ApiService) CreateImage(ctx context.Context, request oapi.CreateImageRe }, nil } } - return oapi.CreateImage201JSONResponse(*img), nil + return oapi.CreateImage202JSONResponse(*img), nil } // GetImage gets image details @@ -70,6 +70,33 @@ func (s *ApiService) GetImage(ctx context.Context, request oapi.GetImageRequestO return oapi.GetImage200JSONResponse(*img), nil } +// GetImageProgress streams build progress via SSE +func (s *ApiService) GetImageProgress(ctx context.Context, request oapi.GetImageProgressRequestObject) (oapi.GetImageProgressResponseObject, error) { + log := logger.FromContext(ctx) + + progressChan, err := s.ImageManager.GetProgress(ctx, request.Id) + if err != nil { + switch { + case errors.Is(err, images.ErrNotFound): + return oapi.GetImageProgress404JSONResponse{ + Code: "not_found", + Message: "image not found", + }, nil + default: + log.Error("failed to get progress", "error", err, "id", request.Id) + return oapi.GetImageProgress500JSONResponse{ + Code: "internal_error", + Message: "failed to get progress", + }, nil + } + } + + // Return SSE stream (uses helper from progress.go) + return oapi.GetImageProgress200TexteventStreamResponse{ + Body: images.ToSSEReader(progressChan), + }, nil +} + // DeleteImage deletes an image func (s *ApiService) DeleteImage(ctx context.Context, request oapi.DeleteImageRequestObject) (oapi.DeleteImageResponseObject, error) { log := logger.FromContext(ctx) diff --git a/cmd/api/api/images_test.go b/cmd/api/api/images_test.go index d948af9c..804bff39 100644 --- a/cmd/api/api/images_test.go +++ b/cmd/api/api/images_test.go @@ -1,8 +1,13 @@ package api import ( + "bufio" + "encoding/json" + "strings" "testing" + "time" + "github.com/onkernel/hypeman/lib/images" "github.com/onkernel/hypeman/lib/oapi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -33,3 +38,117 @@ func TestGetImage_NotFound(t *testing.T) { assert.Equal(t, "image not found", notFound.Message) } +func TestCreateImage_AsyncWithSSE(t *testing.T) { + svc := newTestService(t) + ctx := ctx() + + // 1. Create image (should return 202 Accepted immediately) + createResp, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ + Body: &oapi.CreateImageRequest{ + Name: "docker.io/library/alpine:latest", + }, + }) + require.NoError(t, err) + + acceptedResp, ok := createResp.(oapi.CreateImage202JSONResponse) + require.True(t, ok, "expected 202 accepted response") + + img := oapi.Image(acceptedResp) + require.Equal(t, "docker.io/library/alpine:latest", img.Name) + require.Equal(t, "img-alpine-latest", img.Id) + require.Contains(t, []oapi.ImageStatus{images.StatusPending, images.StatusPulling}, img.Status) + require.Equal(t, 0, img.Progress) + + // 2. Stream progress via SSE + progressResp, err := svc.GetImageProgress(ctx, oapi.GetImageProgressRequestObject{ + Id: img.Id, + }) + require.NoError(t, err) + + sseResp, ok := progressResp.(oapi.GetImageProgress200TexteventStreamResponse) + if !ok { + t.Fatalf("expected SSE stream, got %T", progressResp) + } + + // Read SSE events + scanner := bufio.NewScanner(sseResp.Body) + lastProgress := 0 + sawPulling := false + sawUnpacking := false + sawConverting := false + + timeout := time.After(3 * time.Minute) + done := make(chan bool) + + go func() { + for scanner.Scan() { + line := scanner.Text() + if !strings.HasPrefix(line, "data: ") { + continue + } + + data := strings.TrimPrefix(line, "data: ") + var update images.ProgressUpdate + if err := json.Unmarshal([]byte(data), &update); err != nil { + continue + } + + t.Logf("SSE: status=%s, progress=%d%%", update.Status, update.Progress) + + // Track which phases we see + if update.Status == images.StatusPulling { + sawPulling = true + } + if update.Status == images.StatusUnpacking { + sawUnpacking = true + } + if update.Status == images.StatusConverting { + sawConverting = true + } + + // Progress should be monotonic + require.GreaterOrEqual(t, update.Progress, lastProgress) + lastProgress = update.Progress + + // Stop when ready + if update.Status == images.StatusReady { + require.Equal(t, 100, update.Progress) + done <- true + return + } + + // Fail on error + if update.Status == images.StatusFailed { + t.Fatalf("Build failed: %v", update.Error) + } + } + }() + + // Wait for completion or timeout + select { + case <-done: + // Success + case <-timeout: + t.Fatal("Build did not complete within 3 minutes") + } + + // Verify we saw at least one intermediate phase (build might be too fast to catch all) + sawAnyPhase := sawPulling || sawUnpacking || sawConverting + require.True(t, sawAnyPhase || lastProgress == 100, "should see at least one build phase or final state") + + // 3. Verify final image state + getResp, err := svc.GetImage(ctx, oapi.GetImageRequestObject{Id: img.Id}) + require.NoError(t, err) + + imgResp, ok := getResp.(oapi.GetImage200JSONResponse) + require.True(t, ok, "expected 200 response") + + finalImg := oapi.Image(imgResp) + require.Equal(t, oapi.ImageStatus(images.StatusReady), finalImg.Status) + require.Equal(t, 100, finalImg.Progress) + require.NotNil(t, finalImg.SizeBytes) + require.Greater(t, *finalImg.SizeBytes, int64(0)) + require.Nil(t, finalImg.QueuePosition) + require.Nil(t, finalImg.Error) +} + diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 99c441ac..b57418f3 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -2,18 +2,20 @@ package config import ( "os" + "strconv" "github.com/joho/godotenv" ) type Config struct { - Port string - DataDir string - BridgeName string - SubnetCIDR string - SubnetGateway string - JwtSecret string - DNSServer string + Port string + DataDir string + BridgeName string + SubnetCIDR string + SubnetGateway string + JwtSecret string + DNSServer string + MaxConcurrentBuilds int } // Load loads configuration from environment variables @@ -23,13 +25,14 @@ func Load() *Config { _ = godotenv.Load() cfg := &Config{ - Port: getEnv("PORT", "8080"), - DataDir: getEnv("DATA_DIR", "/var/lib/hypeman"), - BridgeName: getEnv("BRIDGE_NAME", "vmbr0"), - SubnetCIDR: getEnv("SUBNET_CIDR", "192.168.100.0/24"), - SubnetGateway: getEnv("SUBNET_GATEWAY", "192.168.100.1"), - JwtSecret: getEnv("JWT_SECRET", ""), - DNSServer: getEnv("DNS_SERVER", "1.1.1.1"), + Port: getEnv("PORT", "8080"), + DataDir: getEnv("DATA_DIR", "/var/lib/hypeman"), + BridgeName: getEnv("BRIDGE_NAME", "vmbr0"), + SubnetCIDR: getEnv("SUBNET_CIDR", "192.168.100.0/24"), + SubnetGateway: getEnv("SUBNET_GATEWAY", "192.168.100.1"), + JwtSecret: getEnv("JWT_SECRET", ""), + DNSServer: getEnv("DNS_SERVER", "1.1.1.1"), + MaxConcurrentBuilds: getEnvInt("MAX_CONCURRENT_BUILDS", 1), } return cfg @@ -42,3 +45,12 @@ func getEnv(key, defaultValue string) string { return defaultValue } +func getEnvInt(key string, defaultValue int) int { + if value := os.Getenv(key); value != "" { + if intVal, err := strconv.Atoi(value); err == nil { + return intVal + } + } + return defaultValue +} + diff --git a/lib/images/manager.go b/lib/images/manager.go index a6c89470..904598a6 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -7,6 +7,7 @@ import ( "path/filepath" "regexp" "strings" + "sync" "time" "github.com/onkernel/hypeman/lib/oapi" @@ -18,19 +19,29 @@ type Manager interface { CreateImage(ctx context.Context, req oapi.CreateImageRequest) (*oapi.Image, error) GetImage(ctx context.Context, id string) (*oapi.Image, error) DeleteImage(ctx context.Context, id string) error + GetProgress(ctx context.Context, id string) (chan ProgressUpdate, error) + RecoverInterruptedBuilds() } type manager struct { dataDir string ociClient *OCIClient + queue *BuildQueue + trackers map[string]*ProgressTracker + mu sync.RWMutex } // NewManager creates a new image manager with OCI client -func NewManager(dataDir string, ociClient *OCIClient) Manager { - return &manager{ +func NewManager(dataDir string, ociClient *OCIClient, maxConcurrentBuilds int) Manager { + m := &manager{ dataDir: dataDir, ociClient: ociClient, + queue: NewBuildQueue(maxConcurrentBuilds), + trackers: make(map[string]*ProgressTracker), } + // Recover interrupted builds on initialization + m.RecoverInterruptedBuilds() + return m } func (m *manager) ListImages(ctx context.Context) ([]oapi.Image, error) { @@ -60,42 +71,161 @@ func (m *manager) CreateImage(ctx context.Context, req oapi.CreateImageRequest) return nil, ErrAlreadyExists } - // 3. Pull image and export rootfs to temp directory - tempDir := filepath.Join(os.TempDir(), fmt.Sprintf("hypeman-image-%s-%d", *imageID, time.Now().Unix())) - defer os.RemoveAll(tempDir) // cleanup temp dir + // 3. Create initial metadata with pending status + meta := &imageMetadata{ + ID: *imageID, + Name: req.Name, + Status: StatusPending, + Progress: 0, + Request: &req, + CreatedAt: time.Now(), + } + + if err := writeMetadata(m.dataDir, *imageID, meta); err != nil { + return nil, fmt.Errorf("write initial metadata: %w", err) + } + + // 4. Enqueue the build + queuePos := m.queue.Enqueue(*imageID, req, func() { + m.buildImage(context.Background(), *imageID, req) + }) - containerMeta, err := m.ociClient.pullAndExport(ctx, req.Name, tempDir) + meta.QueuePosition = &queuePos + if err := writeMetadata(m.dataDir, *imageID, meta); err != nil { + return nil, fmt.Errorf("update queue position: %w", err) + } + + // 5. Return immediately (build happens in background) + return meta.toOAPI(), nil +} + +// buildImage performs the actual image build in the background +func (m *manager) buildImage(ctx context.Context, imageID string, req oapi.CreateImageRequest) { + // Create progress tracker + tracker := NewProgressTracker(imageID, m.dataDir) + m.registerTracker(imageID, tracker) + defer tracker.Close() + defer m.unregisterTracker(imageID) + defer m.queue.MarkComplete(imageID) + + // Use persistent build directory for resumability + buildDir := filepath.Join(imageDir(m.dataDir, imageID), ".build") + tempDir := filepath.Join(buildDir, "rootfs") + + // Ensure build directory exists + if err := os.MkdirAll(buildDir, 0755); err != nil { + tracker.Fail(fmt.Errorf("create build dir: %w", err)) + return + } + + // Cleanup build dir on success + defer func() { + meta, _ := readMetadata(m.dataDir, imageID) + if meta != nil && meta.Status == StatusReady { + os.RemoveAll(buildDir) + } + }() + + // Phase 1: Pull and unpack (0-90%) + tracker.Update(StatusPulling, 0, nil) + containerMeta, err := m.ociClient.pullAndExportWithProgress(ctx, req.Name, tempDir, tracker) if err != nil { - return nil, fmt.Errorf("pull and export: %w", err) + tracker.Fail(fmt.Errorf("pull and export: %w", err)) + return } - // 5. Convert rootfs directory to ext4 disk image - diskPath := imagePath(m.dataDir, *imageID) + // Phase 2: Convert to ext4 (90-100%) + tracker.Update(StatusConverting, 90, nil) + diskPath := imagePath(m.dataDir, imageID) diskSize, err := convertToExt4(tempDir, diskPath) if err != nil { - return nil, fmt.Errorf("convert to ext4: %w", err) + tracker.Fail(fmt.Errorf("convert to ext4: %w", err)) + return } - // 6. Create metadata - meta := &imageMetadata{ - ID: *imageID, - Name: req.Name, - SizeBytes: diskSize, - Entrypoint: containerMeta.Entrypoint, - Cmd: containerMeta.Cmd, - Env: containerMeta.Env, - WorkingDir: containerMeta.WorkingDir, - CreatedAt: time.Now(), + // Phase 3: Finalize metadata + meta, err := readMetadata(m.dataDir, imageID) + if err != nil { + tracker.Fail(fmt.Errorf("read metadata: %w", err)) + return } - // 7. Write metadata atomically - if err := writeMetadata(m.dataDir, *imageID, meta); err != nil { - // Clean up disk image if metadata write fails - os.Remove(diskPath) - return nil, fmt.Errorf("write metadata: %w", err) + meta.Status = StatusReady + meta.Progress = 100 + meta.QueuePosition = nil + meta.Error = nil + meta.SizeBytes = diskSize + meta.Entrypoint = containerMeta.Entrypoint + meta.Cmd = containerMeta.Cmd + meta.Env = containerMeta.Env + meta.WorkingDir = containerMeta.WorkingDir + + if err := writeMetadata(m.dataDir, imageID, meta); err != nil { + tracker.Fail(fmt.Errorf("write final metadata: %w", err)) + return } - return meta.toOAPI(), nil + tracker.Complete() +} + +// GetProgress returns a channel for SSE progress updates +func (m *manager) GetProgress(ctx context.Context, id string) (chan ProgressUpdate, error) { + // Get or create tracker + m.mu.Lock() + tracker, exists := m.trackers[id] + if !exists { + // Check if image exists before creating tracker + if !imageExists(m.dataDir, id) { + m.mu.Unlock() + return nil, ErrNotFound + } + // No active build, create temporary tracker that sends current state + tracker = NewProgressTracker(id, m.dataDir) + m.trackers[id] = tracker + } + m.mu.Unlock() + + // Subscribe to progress updates + ch, err := tracker.Subscribe(ctx) + if err != nil { + return nil, fmt.Errorf("subscribe to progress: %w", err) + } + + return ch, nil +} + +// RecoverInterruptedBuilds resumes builds that were interrupted by server restart +func (m *manager) RecoverInterruptedBuilds() { + metas, err := listMetadata(m.dataDir) + if err != nil { + return // Best effort + } + + for _, meta := range metas { + switch meta.Status { + case StatusPending, StatusPulling, StatusUnpacking, StatusConverting: + // Re-enqueue the build + if meta.Request != nil { + m.queue.Enqueue(meta.ID, *meta.Request, func() { + m.buildImage(context.Background(), meta.ID, *meta.Request) + }) + } + } + } +} + +// registerTracker adds a tracker to the active map +func (m *manager) registerTracker(imageID string, tracker *ProgressTracker) { + m.mu.Lock() + defer m.mu.Unlock() + m.trackers[imageID] = tracker +} + +// unregisterTracker removes a tracker from the active map +func (m *manager) unregisterTracker(imageID string) { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.trackers, imageID) } func (m *manager) GetImage(ctx context.Context, id string) (*oapi.Image, error) { diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go index fa9a1b45..c4d056ad 100644 --- a/lib/images/manager_test.go +++ b/lib/images/manager_test.go @@ -15,18 +15,37 @@ func TestCreateImage(t *testing.T) { require.NoError(t, err) dataDir := t.TempDir() - mgr := NewManager(dataDir, ociClient) + mgr := NewManager(dataDir, ociClient, 1) ctx := context.Background() req := oapi.CreateImageRequest{ Name: "docker.io/library/alpine:latest", } + // CreateImage returns immediately with pending status img, err := mgr.CreateImage(ctx, req) require.NoError(t, err) require.NotNil(t, img) require.Equal(t, "docker.io/library/alpine:latest", img.Name) require.Equal(t, "img-alpine-latest", img.Id) + require.Contains(t, []oapi.ImageStatus{StatusPending, StatusPulling}, img.Status) + + // Wait for build to complete via progress channel + progressCh, err := mgr.GetProgress(ctx, img.Id) + require.NoError(t, err) + + for update := range progressCh { + if update.Status == StatusReady || update.Status == StatusFailed { + require.Equal(t, StatusReady, update.Status) + break + } + } + + // Verify final state + img, err = mgr.GetImage(ctx, img.Id) + require.NoError(t, err) + require.Equal(t, oapi.ImageStatus(StatusReady), img.Status) + require.Equal(t, 100, img.Progress) require.NotNil(t, img.SizeBytes) require.Greater(t, *img.SizeBytes, int64(0)) @@ -34,11 +53,6 @@ func TestCreateImage(t *testing.T) { diskPath := filepath.Join(dataDir, "images", img.Id, "rootfs.ext4") _, err = os.Stat(diskPath) require.NoError(t, err) - - // Verify metadata file was created - metaPath := filepath.Join(dataDir, "images", img.Id, "metadata.json") - _, err = os.Stat(metaPath) - require.NoError(t, err) } func TestCreateImageWithCustomID(t *testing.T) { @@ -46,7 +60,7 @@ func TestCreateImageWithCustomID(t *testing.T) { require.NoError(t, err) dataDir := t.TempDir() - mgr := NewManager(dataDir, ociClient) + mgr := NewManager(dataDir, ociClient, 1) ctx := context.Background() customID := "my-custom-alpine" @@ -59,6 +73,9 @@ func TestCreateImageWithCustomID(t *testing.T) { require.NoError(t, err) require.NotNil(t, img) require.Equal(t, "my-custom-alpine", img.Id) + + // Wait for build to complete + waitForReady(t, mgr, ctx, img.Id) } func TestCreateImageDuplicate(t *testing.T) { @@ -66,7 +83,7 @@ func TestCreateImageDuplicate(t *testing.T) { require.NoError(t, err) dataDir := t.TempDir() - mgr := NewManager(dataDir, ociClient) + mgr := NewManager(dataDir, ociClient, 1) ctx := context.Background() req := oapi.CreateImageRequest{ @@ -74,10 +91,13 @@ func TestCreateImageDuplicate(t *testing.T) { } // Create first image - _, err = mgr.CreateImage(ctx, req) + img1, err := mgr.CreateImage(ctx, req) require.NoError(t, err) - // Try to create duplicate + // Wait for build to start (moves from pending to pulling) + waitForReady(t, mgr, ctx, img1.Id) + + // Try to create duplicate (should fail even if first still building) _, err = mgr.CreateImage(ctx, req) require.ErrorIs(t, err, ErrAlreadyExists) } @@ -87,7 +107,7 @@ func TestListImages(t *testing.T) { require.NoError(t, err) dataDir := t.TempDir() - mgr := NewManager(dataDir, ociClient) + mgr := NewManager(dataDir, ociClient, 1) ctx := context.Background() @@ -100,14 +120,18 @@ func TestListImages(t *testing.T) { req1 := oapi.CreateImageRequest{ Name: "docker.io/library/alpine:latest", } - _, err = mgr.CreateImage(ctx, req1) + img1, err := mgr.CreateImage(ctx, req1) require.NoError(t, err) + // Wait for build + waitForReady(t, mgr, ctx, img1.Id) + // List should return one image images, err = mgr.ListImages(ctx) require.NoError(t, err) require.Len(t, images, 1) require.Equal(t, "docker.io/library/alpine:latest", images[0].Name) + require.Equal(t, oapi.ImageStatus(StatusReady), images[0].Status) } func TestGetImage(t *testing.T) { @@ -115,7 +139,7 @@ func TestGetImage(t *testing.T) { require.NoError(t, err) dataDir := t.TempDir() - mgr := NewManager(dataDir, ociClient) + mgr := NewManager(dataDir, ociClient, 1) ctx := context.Background() req := oapi.CreateImageRequest{ @@ -125,13 +149,17 @@ func TestGetImage(t *testing.T) { created, err := mgr.CreateImage(ctx, req) require.NoError(t, err) + // Wait for build + waitForReady(t, mgr, ctx, created.Id) + // Get the image img, err := mgr.GetImage(ctx, created.Id) require.NoError(t, err) require.NotNil(t, img) require.Equal(t, created.Id, img.Id) require.Equal(t, created.Name, img.Name) - require.Equal(t, *created.SizeBytes, *img.SizeBytes) + require.Equal(t, oapi.ImageStatus(StatusReady), img.Status) + require.NotNil(t, img.SizeBytes) } func TestGetImageNotFound(t *testing.T) { @@ -139,7 +167,7 @@ func TestGetImageNotFound(t *testing.T) { require.NoError(t, err) dataDir := t.TempDir() - mgr := NewManager(dataDir, ociClient) + mgr := NewManager(dataDir, ociClient, 1) ctx := context.Background() @@ -153,7 +181,7 @@ func TestDeleteImage(t *testing.T) { require.NoError(t, err) dataDir := t.TempDir() - mgr := NewManager(dataDir, ociClient) + mgr := NewManager(dataDir, ociClient, 1) ctx := context.Background() req := oapi.CreateImageRequest{ @@ -163,6 +191,9 @@ func TestDeleteImage(t *testing.T) { created, err := mgr.CreateImage(ctx, req) require.NoError(t, err) + // Wait for ready + waitForReady(t, mgr, ctx, created.Id) + // Delete the image err = mgr.DeleteImage(ctx, created.Id) require.NoError(t, err) @@ -182,7 +213,7 @@ func TestDeleteImageNotFound(t *testing.T) { require.NoError(t, err) dataDir := t.TempDir() - mgr := NewManager(dataDir, ociClient) + mgr := NewManager(dataDir, ociClient, 1) ctx := context.Background() @@ -211,4 +242,20 @@ func TestGenerateImageID(t *testing.T) { } } +// waitForReady waits for an image build to complete +func waitForReady(t *testing.T, mgr Manager, ctx context.Context, imageID string) { + progressCh, err := mgr.GetProgress(ctx, imageID) + require.NoError(t, err) + + for update := range progressCh { + t.Logf("Progress: status=%s, progress=%d%%", update.Status, update.Progress) + if update.Status == StatusReady || update.Status == StatusFailed { + if update.Status == StatusFailed { + t.Fatalf("Build failed: %v", update.Error) + } + break + } + } +} + diff --git a/lib/images/oci.go b/lib/images/oci.go index 2286ceed..0fd258b7 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -10,6 +10,7 @@ import ( "github.com/containers/image/v5/docker" "github.com/containers/image/v5/oci/layout" "github.com/containers/image/v5/signature" + "github.com/containers/image/v5/types" "github.com/opencontainers/image-spec/specs-go/v1" rspec "github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/umoci/oci/cas/dir" @@ -37,6 +38,11 @@ func (c *OCIClient) Close() error { // pullAndExport pulls an OCI image and exports its rootfs to a directory func (c *OCIClient) pullAndExport(ctx context.Context, imageRef, exportDir string) (*containerMetadata, error) { + return c.pullAndExportWithProgress(ctx, imageRef, exportDir, nil) +} + +// pullAndExportWithProgress pulls an OCI image with progress reporting +func (c *OCIClient) pullAndExportWithProgress(ctx context.Context, imageRef, exportDir string, tracker *ProgressTracker) (*containerMetadata, error) { // Create temporary OCI layout directory ociLayoutDir := filepath.Join(c.cacheDir, fmt.Sprintf("oci-layout-%d", os.Getpid())) if err := os.MkdirAll(ociLayoutDir, 0755); err != nil { @@ -44,8 +50,8 @@ func (c *OCIClient) pullAndExport(ctx context.Context, imageRef, exportDir strin } defer os.RemoveAll(ociLayoutDir) - // Pull image to OCI layout - if err := c.pullToOCILayout(ctx, imageRef, ociLayoutDir); err != nil { + // Pull image to OCI layout with progress + if err := c.pullToOCILayoutWithProgress(ctx, imageRef, ociLayoutDir, tracker); err != nil { return nil, fmt.Errorf("pull to oci layout: %w", err) } @@ -55,7 +61,10 @@ func (c *OCIClient) pullAndExport(ctx context.Context, imageRef, exportDir strin return nil, fmt.Errorf("extract metadata: %w", err) } - // Unpack layers to export directory + // Unpack layers to export directory (70-90%) + if tracker != nil { + tracker.Update(StatusUnpacking, 70, nil) + } if err := c.unpackLayers(ctx, ociLayoutDir, exportDir); err != nil { return nil, fmt.Errorf("unpack layers: %w", err) } @@ -65,6 +74,11 @@ func (c *OCIClient) pullAndExport(ctx context.Context, imageRef, exportDir strin // pullToOCILayout pulls an image from a registry to an OCI layout directory func (c *OCIClient) pullToOCILayout(ctx context.Context, imageRef, ociLayoutDir string) error { + return c.pullToOCILayoutWithProgress(ctx, imageRef, ociLayoutDir, nil) +} + +// pullToOCILayoutWithProgress pulls with progress reporting (0-70%) +func (c *OCIClient) pullToOCILayoutWithProgress(ctx context.Context, imageRef, ociLayoutDir string, tracker *ProgressTracker) error { // Parse source reference (docker://...) srcRef, err := docker.ParseReference("//" + imageRef) if err != nil { @@ -86,9 +100,36 @@ func (c *OCIClient) pullToOCILayout(ctx context.Context, imageRef, ociLayoutDir } defer policyContext.Destroy() + // Setup progress reporting if tracker provided + var progressChan chan types.ProgressProperties + if tracker != nil { + progressChan = make(chan types.ProgressProperties) + go func() { + var totalBytes int64 + var downloadedBytes int64 + + for p := range progressChan { + // Track total and downloaded bytes + if p.Event == types.ProgressEventNewArtifact { + totalBytes += p.Artifact.Size + } + if p.Event == types.ProgressEventRead { + downloadedBytes = int64(p.Offset) + } + + // Calculate percentage (0-70% of total progress) + if totalBytes > 0 { + pct := int((float64(downloadedBytes) / float64(totalBytes)) * 70) + tracker.Update(StatusPulling, pct, nil) + } + } + }() + } + // Pull image _, err = copy.Image(ctx, policyContext, destRef, srcRef, ©.Options{ ReportWriter: os.Stdout, + Progress: progressChan, }) if err != nil { return fmt.Errorf("copy image: %w", err) diff --git a/lib/images/progress.go b/lib/images/progress.go new file mode 100644 index 00000000..c6b6a38b --- /dev/null +++ b/lib/images/progress.go @@ -0,0 +1,225 @@ +package images + +import ( + "context" + "encoding/json" + "fmt" + "io" + "sync" +) + +// Build status constants +const ( + StatusPending = "pending" + StatusPulling = "pulling" + StatusUnpacking = "unpacking" + StatusConverting = "converting" + StatusReady = "ready" + StatusFailed = "failed" +) + +// ProgressUpdate represents a status update during image build +type ProgressUpdate struct { + Status string `json:"status"` + Progress int `json:"progress"` + QueuePosition *int `json:"queue_position,omitempty"` + Error *string `json:"error,omitempty"` +} + +// ProgressTracker tracks build progress and broadcasts updates to SSE subscribers +type ProgressTracker struct { + imageID string + dataDir string + subscribers []chan ProgressUpdate + mu sync.RWMutex + closed bool +} + +// NewProgressTracker creates a new progress tracker +func NewProgressTracker(imageID, dataDir string) *ProgressTracker { + return &ProgressTracker{ + imageID: imageID, + dataDir: dataDir, + subscribers: make([]chan ProgressUpdate, 0), + } +} + +// Update updates the progress and broadcasts to all subscribers +func (p *ProgressTracker) Update(status string, progress int, queuePos *int) { + p.mu.RLock() + defer p.mu.RUnlock() + + if p.closed { + return + } + + // Update metadata on disk + meta, err := readMetadata(p.dataDir, p.imageID) + if err != nil { + return // Best effort + } + + meta.Status = status + meta.Progress = progress + meta.QueuePosition = queuePos + writeMetadata(p.dataDir, p.imageID, meta) + + // Broadcast to subscribers + update := ProgressUpdate{ + Status: status, + Progress: progress, + QueuePosition: queuePos, + } + + for _, ch := range p.subscribers { + select { + case ch <- update: + default: + // Non-blocking send (skip slow consumers) + } + } +} + +// Fail marks the build as failed with error message +func (p *ProgressTracker) Fail(err error) { + p.mu.RLock() + defer p.mu.RUnlock() + + if p.closed { + return + } + + meta, metaErr := readMetadata(p.dataDir, p.imageID) + if metaErr != nil { + return + } + + meta.Status = StatusFailed + meta.Progress = 0 + meta.QueuePosition = nil + errorMsg := err.Error() + meta.Error = &errorMsg + writeMetadata(p.dataDir, p.imageID, meta) + + // Broadcast failure + update := ProgressUpdate{ + Status: StatusFailed, + Error: &errorMsg, + } + + for _, ch := range p.subscribers { + select { + case ch <- update: + default: + } + } +} + +// Complete marks the build as complete +func (p *ProgressTracker) Complete() { + p.Update(StatusReady, 100, nil) +} + +// Subscribe adds a new SSE subscriber and returns their channel +func (p *ProgressTracker) Subscribe(ctx context.Context) (chan ProgressUpdate, error) { + p.mu.Lock() + defer p.mu.Unlock() + + if p.closed { + return nil, fmt.Errorf("tracker closed") + } + + ch := make(chan ProgressUpdate, 10) // Buffered for slow consumers + p.subscribers = append(p.subscribers, ch) + + // Send current state immediately + meta, err := readMetadata(p.dataDir, p.imageID) + if err == nil { + update := ProgressUpdate{ + Status: meta.Status, + Progress: meta.Progress, + QueuePosition: meta.QueuePosition, + Error: meta.Error, + } + ch <- update + } + + // Close channel when context is done + go func() { + <-ctx.Done() + p.Unsubscribe(ch) + }() + + return ch, nil +} + +// Unsubscribe removes a subscriber +func (p *ProgressTracker) Unsubscribe(ch chan ProgressUpdate) { + p.mu.Lock() + defer p.mu.Unlock() + + for i, sub := range p.subscribers { + if sub == ch { + p.subscribers = append(p.subscribers[:i], p.subscribers[i+1:]...) + close(ch) + break + } + } +} + +// Close closes all subscriber channels +func (p *ProgressTracker) Close() { + p.mu.Lock() + defer p.mu.Unlock() + + if p.closed { + return + } + + p.closed = true + for _, ch := range p.subscribers { + close(ch) + } + p.subscribers = nil +} + +// ToSSEReader converts a progress channel to an io.ReadCloser for SSE streaming +func ToSSEReader(ch chan ProgressUpdate) io.ReadCloser { + return &sseStream{ch: ch} +} + +// sseStream implements io.ReadCloser for SSE streaming +type sseStream struct { + ch chan ProgressUpdate + buffer []byte +} + +func (s *sseStream) Read(p []byte) (n int, err error) { + // If we have buffered data, return it first + if len(s.buffer) > 0 { + n = copy(p, s.buffer) + s.buffer = s.buffer[n:] + return n, nil + } + + // Get next update from channel + update, ok := <-s.ch + if !ok { + return 0, io.EOF + } + + // Format as SSE + data, _ := json.Marshal(update) + msg := fmt.Sprintf("data: %s\n\n", data) + s.buffer = []byte(msg) + + // Copy to output buffer + n = copy(p, s.buffer) + s.buffer = s.buffer[n:] + return n, nil +} + +func (s *sseStream) Close() error { + return nil +} + diff --git a/lib/images/queue.go b/lib/images/queue.go new file mode 100644 index 00000000..eab2f6f5 --- /dev/null +++ b/lib/images/queue.go @@ -0,0 +1,111 @@ +package images + +import ( + "sync" + + "github.com/onkernel/hypeman/lib/oapi" +) + +// QueuedBuild represents a build waiting in queue +type QueuedBuild struct { + ImageID string + Request oapi.CreateImageRequest + StartFn func() // Callback to start the build +} + +// BuildQueue manages concurrent image builds with a configurable limit +type BuildQueue struct { + maxConcurrent int + active map[string]bool // imageID -> is building + pending []QueuedBuild + mu sync.Mutex +} + +// NewBuildQueue creates a new build queue with max concurrent limit +func NewBuildQueue(maxConcurrent int) *BuildQueue { + if maxConcurrent < 1 { + maxConcurrent = 1 + } + return &BuildQueue{ + maxConcurrent: maxConcurrent, + active: make(map[string]bool), + pending: make([]QueuedBuild, 0), + } +} + +// Enqueue adds a build to the queue and returns queue position +// Returns 0 if build starts immediately, >0 if queued +func (q *BuildQueue) Enqueue(imageID string, req oapi.CreateImageRequest, startFn func()) int { + q.mu.Lock() + defer q.mu.Unlock() + + build := QueuedBuild{ + ImageID: imageID, + Request: req, + StartFn: startFn, + } + + // If under limit, start immediately + if len(q.active) < q.maxConcurrent { + q.active[imageID] = true + go startFn() + return 0 // Building now, not queued + } + + // Otherwise, add to queue + q.pending = append(q.pending, build) + return len(q.pending) // Position in queue +} + +// MarkComplete marks a build as complete and starts the next queued build +func (q *BuildQueue) MarkComplete(imageID string) { + q.mu.Lock() + defer q.mu.Unlock() + + delete(q.active, imageID) + + // Try to start next build + if len(q.pending) > 0 && len(q.active) < q.maxConcurrent { + next := q.pending[0] + q.pending = q.pending[1:] + q.active[next.ImageID] = true + go next.StartFn() + } +} + +// GetPosition returns the queue position for an image +// Returns nil if not in queue (either building or complete) +func (q *BuildQueue) GetPosition(imageID string) *int { + q.mu.Lock() + defer q.mu.Unlock() + + // Check if actively building + if q.active[imageID] { + return nil + } + + // Check if in queue + for i, build := range q.pending { + if build.ImageID == imageID { + pos := i + 1 + return &pos + } + } + + return nil +} + +// ActiveCount returns number of actively building images +func (q *BuildQueue) ActiveCount() int { + q.mu.Lock() + defer q.mu.Unlock() + return len(q.active) +} + +// PendingCount returns number of queued builds +func (q *BuildQueue) PendingCount() int { + q.mu.Lock() + defer q.mu.Unlock() + return len(q.pending) +} + diff --git a/lib/images/storage.go b/lib/images/storage.go index 9408383c..9db8a3fc 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -12,24 +12,37 @@ import ( // imageMetadata represents the metadata stored on disk type imageMetadata struct { - ID string `json:"id"` - Name string `json:"name"` - SizeBytes int64 `json:"size_bytes"` - Entrypoint []string `json:"entrypoint,omitempty"` - Cmd []string `json:"cmd,omitempty"` - Env map[string]string `json:"env,omitempty"` - WorkingDir string `json:"working_dir,omitempty"` - CreatedAt time.Time `json:"created_at"` + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Progress int `json:"progress"` + QueuePosition *int `json:"queue_position,omitempty"` + Error *string `json:"error,omitempty"` + Request *oapi.CreateImageRequest `json:"request,omitempty"` + SizeBytes int64 `json:"size_bytes"` + Entrypoint []string `json:"entrypoint,omitempty"` + Cmd []string `json:"cmd,omitempty"` + Env map[string]string `json:"env,omitempty"` + WorkingDir string `json:"working_dir,omitempty"` + CreatedAt time.Time `json:"created_at"` } // toOAPI converts internal metadata to OpenAPI schema func (m *imageMetadata) toOAPI() *oapi.Image { - sizeBytes := m.SizeBytes img := &oapi.Image{ - Id: m.ID, - Name: m.Name, - SizeBytes: &sizeBytes, - CreatedAt: m.CreatedAt, + Id: m.ID, + Name: m.Name, + Status: oapi.ImageStatus(m.Status), + Progress: m.Progress, + QueuePosition: m.QueuePosition, + Error: m.Error, + CreatedAt: m.CreatedAt, + } + + // Only set size_bytes when ready + if m.Status == StatusReady && m.SizeBytes > 0 { + sizeBytes := m.SizeBytes + img.SizeBytes = &sizeBytes } if len(m.Entrypoint) > 0 { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 4950de53..84acbdf5 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -32,6 +32,16 @@ const ( Ok HealthStatus = "ok" ) +// Defines values for ImageStatus. +const ( + Converting ImageStatus = "converting" + Failed ImageStatus = "failed" + Pending ImageStatus = "pending" + Pulling ImageStatus = "pulling" + Ready ImageStatus = "ready" + Unpacking ImageStatus = "unpacking" +) + // Defines values for InstanceState. const ( Created InstanceState = "Created" @@ -155,14 +165,26 @@ type Image struct { // Env Environment variables from container metadata Env *map[string]string `json:"env,omitempty"` + // Error Error message if status is failed + Error *string `json:"error"` + // Id Unique identifier Id string `json:"id"` // Name OCI image reference Name string `json:"name"` - // SizeBytes Disk size in bytes - SizeBytes *int64 `json:"size_bytes,omitempty"` + // Progress Build progress percentage + Progress int `json:"progress"` + + // QueuePosition Position in build queue (null if not queued) + QueuePosition *int `json:"queue_position"` + + // SizeBytes Disk size in bytes (null until ready) + SizeBytes *int64 `json:"size_bytes"` + + // Status Build status + Status ImageStatus `json:"status"` // Version Image tag or digest Version *string `json:"version"` @@ -171,6 +193,9 @@ type Image struct { WorkingDir *string `json:"working_dir"` } +// ImageStatus Build status +type ImageStatus string + // Instance defines model for Instance. type Instance struct { // CreatedAt Creation timestamp (RFC3339) @@ -397,6 +422,9 @@ type ClientInterface interface { // GetImage request GetImage(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetImageProgress request + GetImageProgress(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // ListInstances request ListInstances(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -515,6 +543,18 @@ func (c *Client) GetImage(ctx context.Context, id string, reqEditors ...RequestE return c.Client.Do(req) } +func (c *Client) GetImageProgress(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetImageProgressRequest(c.Server, id) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) ListInstances(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewListInstancesRequest(c.Server) if err != nil { @@ -869,6 +909,40 @@ func NewGetImageRequest(server string, id string) (*http.Request, error) { return req, nil } +// NewGetImageProgressRequest generates requests for GetImageProgress +func NewGetImageProgressRequest(server string, id string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/images/%s/progress", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewListInstancesRequest generates requests for ListInstances func NewListInstancesRequest(server string) (*http.Request, error) { var err error @@ -1434,6 +1508,9 @@ type ClientWithResponsesInterface interface { // GetImageWithResponse request GetImageWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetImageResponse, error) + // GetImageProgressWithResponse request + GetImageProgressWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetImageProgressResponse, error) + // ListInstancesWithResponse request ListInstancesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*ListInstancesResponse, error) @@ -1529,7 +1606,7 @@ func (r ListImagesResponse) StatusCode() int { type CreateImageResponse struct { Body []byte HTTPResponse *http.Response - JSON201 *Image + JSON202 *Image JSON400 *Error JSON401 *Error JSON500 *Error @@ -1598,6 +1675,29 @@ func (r GetImageResponse) StatusCode() int { return 0 } +type GetImageProgressResponse struct { + Body []byte + HTTPResponse *http.Response + JSON404 *Error + JSON500 *Error +} + +// Status returns HTTPResponse.Status +func (r GetImageProgressResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetImageProgressResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type ListInstancesResponse struct { Body []byte HTTPResponse *http.Response @@ -1966,6 +2066,15 @@ func (c *ClientWithResponses) GetImageWithResponse(ctx context.Context, id strin return ParseGetImageResponse(rsp) } +// GetImageProgressWithResponse request returning *GetImageProgressResponse +func (c *ClientWithResponses) GetImageProgressWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetImageProgressResponse, error) { + rsp, err := c.GetImageProgress(ctx, id, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetImageProgressResponse(rsp) +} + // ListInstancesWithResponse request returning *ListInstancesResponse func (c *ClientWithResponses) ListInstancesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*ListInstancesResponse, error) { rsp, err := c.ListInstances(ctx, reqEditors...) @@ -2187,12 +2296,12 @@ func ParseCreateImageResponse(rsp *http.Response) (*CreateImageResponse, error) } switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 201: + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 202: var dest Image if err := json.Unmarshal(bodyBytes, &dest); err != nil { return nil, err } - response.JSON201 = &dest + response.JSON202 = &dest case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: var dest Error @@ -2293,6 +2402,39 @@ func ParseGetImageResponse(rsp *http.Response) (*GetImageResponse, error) { return response, nil } +// ParseGetImageProgressResponse parses an HTTP response from a GetImageProgressWithResponse call +func ParseGetImageProgressResponse(rsp *http.Response) (*GetImageProgressResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetImageProgressResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + // ParseListInstancesResponse parses an HTTP response from a ListInstancesWithResponse call func ParseListInstancesResponse(rsp *http.Response) (*ListInstancesResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -2851,6 +2993,9 @@ type ServerInterface interface { // Get image details // (GET /images/{id}) GetImage(w http.ResponseWriter, r *http.Request, id string) + // Stream image build progress (SSE) + // (GET /images/{id}/progress) + GetImageProgress(w http.ResponseWriter, r *http.Request, id string) // List instances // (GET /instances) ListInstances(w http.ResponseWriter, r *http.Request) @@ -2926,6 +3071,12 @@ func (_ Unimplemented) GetImage(w http.ResponseWriter, r *http.Request, id strin w.WriteHeader(http.StatusNotImplemented) } +// Stream image build progress (SSE) +// (GET /images/{id}/progress) +func (_ Unimplemented) GetImageProgress(w http.ResponseWriter, r *http.Request, id string) { + w.WriteHeader(http.StatusNotImplemented) +} + // List instances // (GET /instances) func (_ Unimplemented) ListInstances(w http.ResponseWriter, r *http.Request) { @@ -3129,6 +3280,37 @@ func (siw *ServerInterfaceWrapper) GetImage(w http.ResponseWriter, r *http.Reque handler.ServeHTTP(w, r) } +// GetImageProgress operation middleware +func (siw *ServerInterfaceWrapper) GetImageProgress(w http.ResponseWriter, r *http.Request) { + + var err error + + // ------------- Path parameter "id" ------------- + var id string + + err = runtime.BindStyledParameterWithOptions("simple", "id", chi.URLParam(r, "id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) + return + } + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.GetImageProgress(w, r, id) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + // ListInstances operation middleware func (siw *ServerInterfaceWrapper) ListInstances(w http.ResponseWriter, r *http.Request) { @@ -3653,6 +3835,9 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/images/{id}", wrapper.GetImage) }) + r.Group(func(r chi.Router) { + r.Get(options.BaseURL+"/images/{id}/progress", wrapper.GetImageProgress) + }) r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/instances", wrapper.ListInstances) }) @@ -3754,11 +3939,11 @@ type CreateImageResponseObject interface { VisitCreateImageResponse(w http.ResponseWriter) error } -type CreateImage201JSONResponse Image +type CreateImage202JSONResponse Image -func (response CreateImage201JSONResponse) VisitCreateImageResponse(w http.ResponseWriter) error { +func (response CreateImage202JSONResponse) VisitCreateImageResponse(w http.ResponseWriter) error { w.Header().Set("Content-Type", "application/json") - w.WriteHeader(201) + w.WriteHeader(202) return json.NewEncoder(w).Encode(response) } @@ -3859,6 +4044,51 @@ func (response GetImage500JSONResponse) VisitGetImageResponse(w http.ResponseWri return json.NewEncoder(w).Encode(response) } +type GetImageProgressRequestObject struct { + Id string `json:"id"` +} + +type GetImageProgressResponseObject interface { + VisitGetImageProgressResponse(w http.ResponseWriter) error +} + +type GetImageProgress200TexteventStreamResponse struct { + Body io.Reader + ContentLength int64 +} + +func (response GetImageProgress200TexteventStreamResponse) VisitGetImageProgressResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "text/event-stream") + if response.ContentLength != 0 { + w.Header().Set("Content-Length", fmt.Sprint(response.ContentLength)) + } + w.WriteHeader(200) + + if closer, ok := response.Body.(io.ReadCloser); ok { + defer closer.Close() + } + _, err := io.Copy(w, response.Body) + return err +} + +type GetImageProgress404JSONResponse Error + +func (response GetImageProgress404JSONResponse) VisitGetImageProgressResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + + return json.NewEncoder(w).Encode(response) +} + +type GetImageProgress500JSONResponse Error + +func (response GetImageProgress500JSONResponse) VisitGetImageProgressResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(500) + + return json.NewEncoder(w).Encode(response) +} + type ListInstancesRequestObject struct { } @@ -4405,6 +4635,9 @@ type StrictServerInterface interface { // Get image details // (GET /images/{id}) GetImage(ctx context.Context, request GetImageRequestObject) (GetImageResponseObject, error) + // Stream image build progress (SSE) + // (GET /images/{id}/progress) + GetImageProgress(ctx context.Context, request GetImageProgressRequestObject) (GetImageProgressResponseObject, error) // List instances // (GET /instances) ListInstances(ctx context.Context, request ListInstancesRequestObject) (ListInstancesResponseObject, error) @@ -4606,6 +4839,32 @@ func (sh *strictHandler) GetImage(w http.ResponseWriter, r *http.Request, id str } } +// GetImageProgress operation middleware +func (sh *strictHandler) GetImageProgress(w http.ResponseWriter, r *http.Request, id string) { + var request GetImageProgressRequestObject + + request.Id = id + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.GetImageProgress(ctx, request.(GetImageProgressRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "GetImageProgress") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(GetImageProgressResponseObject); ok { + if err := validResponse.VisitGetImageProgressResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + // ListInstances operation middleware func (sh *strictHandler) ListInstances(w http.ResponseWriter, r *http.Request) { var request ListInstancesRequestObject @@ -4963,57 +5222,61 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xcbW8TuRb+K5bv/VCkpEnawkLuJyjsUmkLFYWudFkUOeOTjBePPdieQED971d+mcxM", - "xnkptFmyt9JK22Q859iPn/Picxy+4URmuRQgjMbDb1gnKWTE/fnUGJKkV5IXGbyBTwVoY7/OlcxBGQZu", - "UCYLYUY5Man9REEniuWGSYGH+IKYFH1OQQGaOSlIp7LgFI0BufeA4g6GLyTLOeAh7mXC9CgxBHewmef2", - "K20UE1N83cEKCJWCz72aCSm4wcMJ4Ro6S2rPrWhENLKvdN07C3ljKTkQga+dxE8FU0Dx8H19GR8Wg+X4", - "L0iMVX6qgBg4y8h0NRKMthF47f4gHCWFNjJDjIIwbMJAoQNSGNmdggBFDFDEJkhIg3IlZ4wCfdBAhmXT", - "rpgy8aU7G8TAESSDiPbTM8TsnJGCCSgQCaADOJwedhCVyUdQh0z2OBsrouY9J37IiQFtmsrXj21PZwla", - "N7c1oAptiEhW4wpiZv9HKGUezIvG4xYWTQxeiBlTUmQgDJoRxciYg64v7xt+9fr5i9GLV1d4aDXTInGv", - "dvDF6zdv8RAf9/t9K7c1/9iGvxPsUwH1fZ5IhUwKiIV1ooNyi9F4jhLCOailzRbadMk4GRwdx/ba7Whb", - "syNnTXGLP0mqZAYrCJRBJtV8lJEvo2zcMLGT/pNHLQsjX1hWZMi/hT4zk6JUmpwXU8QEOn9WV+4FBI1M", - "GJiCqqtsqhv0j06W1T0jGkpdLfFH/ZPHMfFxk3hZZER0rWOwREBuUB2obN79LNVHLgntRoHKpTKjjOQ5", - "E1MdcXlSGVQ+RhMlM5RKbZCRaFp4a2EGMvfmvxVM8BD/q1d54F5wvz0r59yLqXGPKEXm7jPLQBZmpCGR", - "guoGgseP+v1lBN/68Y6MOiEcukZ2v4KSSENGhGFJwyZ+ObIi2pjOkrxoKjta1vSqyMagkJygGVOmIByd", - "XrxrCD+KSnbxIQKoDz/aAkhcPNoWQf+ij2HW+tswLvkpZoNRIIS3sdVOa0NQvMtQMJO8a2Nk9waBwE83", - "ynYnyqMfk6fZVxhNx22Rl+yr9WloyqZkPDdNnzqIsCcWFSr5MahfKCVVG9xE0sgSn+Y5Zwmxn7o6h4RN", - "WILASkD2BXSQkSRlAha230R1TOhIhe3sxGKKIYxH6Pl0EZWCsjASHVhTywpuWM7BP9MPtuWuW/lzJylm", - "/UwIUCMo4bmBpAy0jkaPJb9YrmUxxHkOCuNiOrWQ1KE7Z1ozMUXl7qIJA06HPvPYmB243awmtpIHYQ1b", - "suF3+RlUl8MMeJ0E3qLsZDOpAC144jdtKQjPCGd0xERemHjEXAHlr4UyKZRMQGRsHa/NAPyG1ZX4mG1t", - "fSILQaNgteB4CYT7bLuJhDbEFCFjKjKLrfxo8azUyY8btyMIiW3DWZl3LG1AFnF2p+fPffBLpDCECVAo", - "A0NCbr+Y0XvsMkncwV3LKUogkwLJyeQ/dgYLU2l7uYJzy1M8NKqAtoEkzknTETGRqdlnltE2hmpDshwd", - "vPn19Pj4+EnTJRz1jx52+4Pu4OHbQX/Yt//9F3fwRKrMysWUGOhaITF2gDBqnksmIjN4sXi2HUY9n4B3", - "K5mHOv0xgO4gp95mLd/wxdO3L+1Jr9Cqx2VCeE+PmRjWPi8+Vg/cH/7jmInvz8Xv5kj1Q2elEAF9BG1p", - "e870R6RDpG1F2YfHjx7/0n8yOKqRkgnz6ARHMyxQ2kmNHxwMmSLr4tnUz7Na0mLqK0hVLcVmzkxMR5Sp", - "tpo//ENEmYLE2FR+C8LgHsnzzarXZHI1VxB1auFUFvFrt+9Djm/qQ+7i4NuCYPKJilgU43yOPhWEW9Oh", - "iMqMMNFOIWuH1UNnwNsQJSV6pAXJdSoj6P6RggugBJVjEHxh2uhwlmZ6cZiuTyXUgpYLPd/lG36OE3iD", - "clJM2LSw+VXWPH1/74F7hfTxnh62b+lknSs2IwZGLI/o88/Q2QUilCrQDXeMB0+ODgePHh8O+v3DQX8b", - "O9CGqFU+5tI++w4H83Clg9lmOgY24Vd6zEs32L0l83zlImR+ozUcbXCSG9cQrYzESiFJoDwJ1b6bFD/u", - "suDhKxZAUTlid/WOkgFbR83LkjBLjrAsdTpxwz9FF/myCR2iq/NzFKSjcWHc4SeYATo45bKg6OU8BzVj", - "WiokiGEzeGAlvCmEYGJqJSCmEUnsEz5Hyn+//uULUmiv3b6bu0/r37hMC0PlZ+He0WlhkP3kpmyXEALS", - "ehHeMIbolXTvhJl2kJDLkc0PJ4KO5+3hy1HwICECjW0Gqo1UQB/86fgbDn0BadzBATHcwX75uIPLVdk/", - "/ezcX05xbacrc6p7y1aK5CqaI+ukV/hmJtzB141DV+d1o3gctbFUrhcovUA7rCksLi5X0shE8kbJEpsk", - "r+HlPxU0j6x/yWKq2XXqa49ZiLfGNmQkWPfIyDV2c/YcsQkqx65JTTa6w9vOYvtPbnoSvnn2tb7Cua7h", - "6Dt/9tlK/Oo9xo3o7U01te7LSyUbvXgrYvxgc5fpsqvbMPy7aPGWR4S25nU93zLqjmKcDLu6hpOrDgRL", - "e1Hp6KxvK1tCQFIoZuaXNoh7zMdAFKinhcfcRXe3CPd1pTw1JsfX164aPIn4kt9AgGIJenpx5o5NGRFk", - "auPk1TnibALJPOGACle5bQUx19B7fXrWtYcBisok3R0fmXGA2NEZEVY+rhUXcP9wcOjapTIHQXKGh/jY", - "fdXBFga3xF66KGFOwdHOks75ojPq5m5CkdMiq3MptMfmqN/3NV9hAl9JVfbv/aV9ecNnRJvypaDBQbhk", - "jBaGxLHKT9TnTrrIMqLmdu3uW5SkkHx0j3ouf9IrF/Q70+bMD/nBFW2VCvpKbTv/a63UzstmrmH61x18", - "0h/cGsK+fxNR+06QwqRSsa9ArdKHt7itK5WeCQNKEI40qBmoUI2vGyEevm+a3/sP1x/q++7gqrDKpY7s", - "de2qCPaOAbR5Jun81pYYuYxy3XRCNpxdt5h2ezsbCBYB2ZVAQtzxfNrB1j4jFJXNu3sOr+fwRcE5IoLa", - "8+8MlEGLinbdk/W+MXrtgwoHf8Rrsvy5+75keU4UycCA0k4/s3N1Qa9MoXyC0iRop4bGckT90CLvyap6", - "m59hINvJDvZgqWe3R3vvN63c7c7K4LvDbe3vyieVHfx7mmykyW8QolwFmvMM4XS6Ic1ZjNpJplP2AW6S", - "7CxmeB8rtsl36nCtTXmqnswdZj1Lt0V3nfgs+BYDPFRv7tOfn5fSnkUuAXJV56qT2PRx2yZAFef/phyo", - "JN3O06BS8V6GONeXsiSgISWqxZGVWdFO97q/W5+18/Ror+njMqQWdG0H0uNyqteVuUoYfpeugX3rvOq0", - "rnVIzuVnZOeFDrRRQDJf7bu8fOGup9pBnwpQ80rnxL2D63qWq7PtX9esbpNyJvx9bgWmUMJfAwJ3+zGm", - "PdzMjOgexPq0W5iSgS+mBzMQpusRaJIqcgXTvpBzwsT6ke2UU05RUHFvWNv5ZcfIhW15njpuxswrdEBd", - "4yKamb7xA/7RrrtsA//NFDvpP7l71adSTDhLDOpWHLGzYMKmc4KO50iqen99n8gfyFqtzHnGsK4o/8tn", - "K/kfWvv/aP5Xe/9/bgGJVAoS42/d7FdNupZO1Uz5wF3UqS7AdMp0/er8PB4Qwp2p3jf/x9mmM1z1A+c7", - "yr4iQsqp7YWVha44hXCVYucWJhdN/j0tuVvgyiU4h14/a8a9dv2H9/vAy9sv9sX+6YGtSn07tYrFBaOf", - "xSp2HYHCHAhXQOi8gce+GKhnWrkSI5cKgrVruitbHleLi7p33/AITuEG7Y5yBfeV4S2aHTWw1rU6Fq75", - "7hod3+H7bm9zS5at9Hz3LY6fvsUxK/ew8mJbNjXuLvHYqqWxSDl329C4+nniKdN7GUrD9ZLZIkStqnrv", - "kmD93TnFXfdQrvb4XPQblMG21j9xAqxET4flWnpCOKIwAy5z96tWPxZ3cKF4uBE97PlfkadSG/erEHz9", - "4fp/AQAA//+BHkAF40wAAA==", + "H4sIAAAAAAAC/+xce28TuRb/Kpbv/aNISfNoYSH3LyjsUolCRdmudFkUOeOTxFuPPbU9gYD63a/8mMlM", + "xnkU2izZWwmJTMY+xz7ndx4+x+k3nMg0kwKE0XjwDetkCilxH58bQ5LppeR5Cu/hOgdt7NeZkhkow8AN", + "SmUuzDAjZmqfKOhEscwwKfAAnxMzRZ+noADNHBWkpzLnFI0AuXlAcQvDF5JmHPAAd1JhOpQYglvYzDP7", + "lTaKiQm+aWEFhErB557NmOTc4MGYcA2tJbZnljQiGtkpbTenpDeSkgMR+MZRvM6ZAooHH6vb+FQOlqO/", + "IDGW+YkCYuA0JZPVkmC0KYF37gPhKMm1kSliFIRhYwYKHZDcyPYEBChigCI2RkIalCk5YxToo5pkWDpp", + "iwkTX9qzXkw4gqQQ4X5yiphdM1IwBgUiAXQAh5PDFqIyuQJ1yGSHs5Eiat5x5AecGNCmznz92OZylkTr", + "1rZGqEIbIpLVcgUxs/8RSpkX5nntdUMWdRm8EjOmpEhBGDQjipERB13d3jf89t3LV8NXby/xwHKmeeKm", + "tvD5u/cf8AAfdbtdS7ex/pjCfxfsOoeqnsdSITMFxMI+0UGhYjSao4RwDmpJ2UKbNhklvf5RTNdOo03O", + "DpwVxg38JFMlU1gBoBRSqebDlHwZpqOaiR13nz1pWBj5wtI8RX4W+szMFE2lyXg+QUygsxdV5p5A4MiE", + "gQmoKss6u163f7zM7gXRUPBqkO93j5/GyMdN4nWeEtG2jsECAblBVUGl8/Znqa64JLQdFVQmlRmmJMuY", + "mOiIy5PKoOI1GiuZoqnUBhmJJrm3FmYgdTP/rWCMB/hfnYUH7gT327F0zjyZCvaIUmTunlkKMjdDDYkU", + "VNckePSk212W4Ac/3oFRJ4RD28j2V1ASaUiJMCyp2cQvfUuiKdNZkuV1Zv1lTm/zdAQKyTGaMWVywtHJ", + "+e814v0oZRcfIgL14UdbARIXj7aVoJ/oY5i1/qYYl/wUs8EoAMLb2GqntSEo3mcomEnetjGyfYtA4Jcb", + "Rbsj5aUfo6fZVxhORk2SF+yr9WlowiZkNDd1n9qLoCcWFRb0Y6J+pZRUTeEmkka2+DzLOEuIfWrrDBI2", + "ZgkCSwHZCeggJcmUCShtvy7VEaFDFdTZisUUQxiPwPN5GZUCszASHVhTS3NuWMbBv9OPtsWu2/lLRylm", + "/UwIUEMoxHMLSiloHY0eS36x2Es5xHkOCqN8MrEiqYrujGnNxAQV2kVjBpwOfOaxMTtw2lwsbCUOwh62", + "RMMb+RlUm8MMeBUE3qLsYlOpAJU48UpbCsIzwhkdMpHlJh4xV4jy11yZKRRIQGRkHa/NALzCqkx8zLa2", + "Ppa5oFFhNcTxGgj32XZdEtoQk4eMKU+tbOWVleeCnbzaqI5AJKaG0yLvWFJAGnF2J2cvffBLpDCECVAo", + "BUNCbl+u6CN2mSRu4bbFFCWQSoHkePwfu4LSVJpeLufc4hQPjMqhaSCJc9J0SExkafadRbSNodqQNEMH", + "7389OTo6elZ3Cf1u/3G722v3Hn/odQdd+++/uIXHUqWWLqbEQNsSiaEDhFHzTDIRWcGr8t12Mur4BLy9", + "oHmopz8moHvIqbfZyzd8/vzDa3vSy7XqcJkQ3tEjJgaV5/Jx8cJ98I8jJqK5eOkMl1bqbD+Yqo2rHt+I", + "aTQmjC+dP7Oc8/D9wO5EQFIiRTovsEKuleR8mzPB/RztfujM1rJWPVGgIzHuRc44RcV7lIFKQBjv3Csh", + "v9vCqT8XFE9M+KdoLnmdQw7DTGrm2TQzaf/GJhkjtwI3Ax1YHRQpkvuqniD1V2qpwtylHT5taTB+yfQV", + "0iG9cWMCz1wYxl1RYV7j+PjoydNfus96/YpzYMI8OcZbLaV02zGph7et0qdnIKiPwBat/lMuMpJc+c+J", + "FDNryO7BrdX6LI/1WjAo3jWAMAOloxrx0cqQCbK5AJt4IC0oltjaaCX2iMXEZEhZxGb/8C8RZQoSY898", + "W3gW3CFZtpn1mpS/lHRpCLU4Eo2I4UgfCYp3H4CObhuA7qNq0hDB+JqKWArE+Rxd54Rbf0cRlSlhonn+", + "qFQ6Dp333wY8U6KHWpBMT2VEun9MwWVfBBVjEHxh2uhQiGG6rMRUlxIKictVwu9y6D9H+aYGOSnGbJLb", + "5Dytl26+t1qzgvpoTys1d1SWyRSbEQNDlkX4+Xfo9BwRSoODWWyn96x/2Hvy9LDX7R72utvYgTZErfIx", + "F/bddziYxysdzDbLMbBJfoXHvHCD3SyZZSs3IbNb7aG/wUlu3EO0rBaroyUB8iSUim9TObvPapkvdwFF", + "xYjdFcsKBGwdNS8KwCw5wqJO7sgN/hRt5GtudIAuz85QoI5GuXEpYDADdHDCZU7R63kGasa0VEgQw2bw", + "yFJ4nwvBxMRSsLk/SewbPkfKf79+8jnJtedu52buaf2Mi2luqPws3Bw9zQ2yT27JdgshIK0n4Q1jgN5K", + "NyestIWEXI5sfjgRdDRvDl+OggcJEWhkjw3aSAX00Z+ikl0GSeMWDhLDLey3j1u42JX96FfnPjnGFU0v", + "zKnqLRspkiuHD62TXuGbmXBVEzcOXZ5VjeJp1Mamcj1B6QnaYXVicXKZkkYmktfq3dgkWUVe/imnWWT/", + "SxazWF2ruveYhXhrbIqMBOseGrnGbk5f2tNRMXZNarLRHd51Ftt9dtsyyu2zr/Xl8XXdat82tu9Wyq/a", + "oN4ovb0pxddOQYHJRi/eiBg/eDOA6eJKQM3w7+N+QHFEaHJed2GgiLrDGCaDVtdgctWBYEkXCx6t9XcS", + "LCAgyRUz8wsbxL3MR0AUqOe5l7mL7m4T7usF86kxGb65ca2EccSX/AYCFEvQ8/NTd2xKiSATGycvzxBn", + "Y0jmCQeUu7J/I4i5bvC7k9O2PQxQVCTp7vjIjBOIHZ0SYenjSsEBdw97h67XLjMQJGN4gI/cVy1sxeC2", + "2JmW9e8JONhZ0DlfdErd2k2okFvJ6kwK7WXT73Z9w0CYgFey6Bl1/tK+5OEzok35UuDgRLhkjFYMiUOV", + "X6jPnXSepkTN7d7dtyiZQnLlXnVc/qRXbugN0+bUD/nBHW2VCvoyfzP/a+zUrstmrmH5Ny183O3dmYR9", + "8y/C9ndBcjOVin0Fapk+vkO1rmR6KgwoQTjSoGagQiunaoR48LFufh8/3Xyq6t2JayGrTOqIriv3jLB3", + "DKDNC0nnd7bFyE2mm7oTsuHspoG0/p2tIAAsImRXAhkVdU+f1RM9F8kjj64dKPoFoajoAz8gej2iz3PO", + "EREUhZozKpsSVb/W+cbojQ8xHPyBr475l+77AvMZUSQFA0o7/syu1YXAIqHy6Uodrq2KNJbj66cGlI9X", + "Vd/8CqlX/PEOdLDU/t0j3XulFdpurQzFO1Rrd1ceqrgM8gCTjTD5DULMWwhtyTN0qn3HtSg6X/Rl/gY0", + "GfhiOjADYdraKCBpXcDLBBvCLFaPHA0UaDxAaCOELpykAopG9Vb0wcXFq0cBUqH8sSGPLkftJJUuGk23", + "yabLFT6kH9sk1FVxrc2pF02/e0yrl+6yb5VZ352KF3iLCTyUB0Nd5yGj/gkh7VHkcmp3AFq0qus+btuc", + "eoH5vymtLkC388y6YLynIU9mDgQ0ZNmVOLIyRdqprru79Vk7z7j3Gj4u6W6IrulAOlxO1qfdYfgbObmX", + "rLvVuDckOZefkV0XOvApsi8nuzyv5Vle56DmC55jNwdX+SyX/5u//Vvdh+dM+F+bKDC5Ev7uGbi72THu", + "4d54hHcvdhHg7g8aLT8h44SJWx5J3sjJzs8he+6X/VGk2ITHaeQM4s0rtNhdZyyamb73A/7Rrru4Z/A3", + "Q+y4++z+WZ9IMeYsMai9wIhdBRM2nRN0NEdSVS9w7BP4A1gXO3OeMewriv/i3Ur8h7sj/2j8L3T/f24B", + "iVQKEuOvde1Xm6OSTlVM+cDdBFvcsGoV6frl2Vk8IIRLeZ1v/sPppjPc4s8v3FP2FSFSLG0vrCxcu6AQ", + "7urs3MJkeYtkT7s4VnDFFpxDr5414167+mdB9gGXd1/si/1hlK1KfTu1ivIG289iFbuOQGENhLufOtXk", + "sS8G6pFW7MTIpYJg5R74ypbHZXkT/P4bHsEp3KLdUezgoTK8RbOjIqx1rY7SNd9fo+M7fN/dKbdA2UrP", + "99Di+OlbHLNChwsvtmVT4/4Sj61aGmXKuduGxuXPE0+Z3stQGm4szcoQtarqvUuAdXfnFHfdQ7nc43PR", + "b1AE20r/xBGwFD0clmvpCeGIwgy4zNzPpv1Y3MK54uHK/aDj/8bFVGrjfnaEbz7d/C8AAP//4IBc+4FR", + "AAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 888696df..33a785c0 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -38,7 +38,7 @@ func ProvideOCIClient(cfg *config.Config) (*images.OCIClient, error) { // ProvideImageManager provides the image manager func ProvideImageManager(cfg *config.Config, ociClient *images.OCIClient) images.Manager { - return images.NewManager(cfg.DataDir, ociClient) + return images.NewManager(cfg.DataDir, ociClient, cfg.MaxConcurrentBuilds) } // ProvideInstanceManager provides the instance manager diff --git a/openapi.yaml b/openapi.yaml index 37b24c75..5b794efd 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -240,7 +240,7 @@ components: Image: type: object - required: [id, name, created_at] + required: [id, name, status, progress, created_at] properties: id: type: string @@ -250,6 +250,27 @@ components: type: string description: OCI image reference example: docker.io/library/nginx:latest + status: + type: string + enum: [pending, pulling, unpacking, converting, ready, failed] + description: Build status + example: ready + progress: + type: integer + minimum: 0 + maximum: 100 + description: Build progress percentage + example: 100 + queue_position: + type: integer + description: Position in build queue (null if not queued) + example: 2 + nullable: true + error: + type: string + description: Error message if status is failed + example: "pull failed: connection timeout" + nullable: true version: type: string description: Image tag or digest @@ -258,8 +279,9 @@ components: size_bytes: type: integer format: int64 - description: Disk size in bytes + description: Disk size in bytes (null until ready) example: 536870912 + nullable: true entrypoint: type: array items: @@ -415,8 +437,8 @@ paths: schema: $ref: "#/components/schemas/CreateImageRequest" responses: - 201: - description: Image created + 202: + description: Image build started (async) content: application/json: schema: @@ -440,6 +462,38 @@ paths: schema: $ref: "#/components/schemas/Error" + /images/{id}/progress: + get: + summary: Stream image build progress (SSE) + operationId: getImageProgress + security: + - bearerAuth: [] + parameters: + - name: id + in: path + required: true + schema: + type: string + responses: + 200: + description: Progress event stream + content: + text/event-stream: + schema: + type: string + 404: + description: Image not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 500: + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /images/{id}: get: summary: Get image details From c376477a01ec6a87a906bf3f2139d2a4e9a69a3a Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 7 Nov 2025 15:16:05 -0500 Subject: [PATCH 12/37] Simplify async API: remove sse for now --- README.md | 9 + cmd/api/api/images.go | 27 --- cmd/api/api/images_test.go | 154 ----------------- lib/images/README.md | 13 +- lib/images/manager.go | 128 ++++++-------- lib/images/manager_test.go | 48 +++--- lib/images/oci.go | 50 +----- lib/images/progress.go | 225 ------------------------ lib/images/storage.go | 66 +++----- lib/oapi/oapi.go | 339 ++++++------------------------------- openapi.yaml | 40 +---- 11 files changed, 170 insertions(+), 929 deletions(-) delete mode 100644 cmd/api/api/images_test.go delete mode 100644 lib/images/progress.go diff --git a/README.md b/README.md index 626ec487..286fbc10 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,15 @@ sudo mkdir /var/lib/hypeman sudo chown $USER:$USER /var/lib/hypeman ``` +#### Dockerhub login + +Requires Docker Hub authentication to avoid rate limits when running the tests: +```bash +docker login +``` + +Docker itself isn't required to be installed. `~/.docker/config.json` is a standard used for handling registry authentication. + ### Build ```bash diff --git a/cmd/api/api/images.go b/cmd/api/api/images.go index ae265d80..e1419976 100644 --- a/cmd/api/api/images.go +++ b/cmd/api/api/images.go @@ -70,33 +70,6 @@ func (s *ApiService) GetImage(ctx context.Context, request oapi.GetImageRequestO return oapi.GetImage200JSONResponse(*img), nil } -// GetImageProgress streams build progress via SSE -func (s *ApiService) GetImageProgress(ctx context.Context, request oapi.GetImageProgressRequestObject) (oapi.GetImageProgressResponseObject, error) { - log := logger.FromContext(ctx) - - progressChan, err := s.ImageManager.GetProgress(ctx, request.Id) - if err != nil { - switch { - case errors.Is(err, images.ErrNotFound): - return oapi.GetImageProgress404JSONResponse{ - Code: "not_found", - Message: "image not found", - }, nil - default: - log.Error("failed to get progress", "error", err, "id", request.Id) - return oapi.GetImageProgress500JSONResponse{ - Code: "internal_error", - Message: "failed to get progress", - }, nil - } - } - - // Return SSE stream (uses helper from progress.go) - return oapi.GetImageProgress200TexteventStreamResponse{ - Body: images.ToSSEReader(progressChan), - }, nil -} - // DeleteImage deletes an image func (s *ApiService) DeleteImage(ctx context.Context, request oapi.DeleteImageRequestObject) (oapi.DeleteImageResponseObject, error) { log := logger.FromContext(ctx) diff --git a/cmd/api/api/images_test.go b/cmd/api/api/images_test.go deleted file mode 100644 index 804bff39..00000000 --- a/cmd/api/api/images_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package api - -import ( - "bufio" - "encoding/json" - "strings" - "testing" - "time" - - "github.com/onkernel/hypeman/lib/images" - "github.com/onkernel/hypeman/lib/oapi" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestListImages_Empty(t *testing.T) { - svc := newTestService(t) - - resp, err := svc.ListImages(ctx(), oapi.ListImagesRequestObject{}) - require.NoError(t, err) - - list, ok := resp.(oapi.ListImages200JSONResponse) - require.True(t, ok, "expected 200 response") - assert.Empty(t, list) -} - -func TestGetImage_NotFound(t *testing.T) { - svc := newTestService(t) - - resp, err := svc.GetImage(ctx(), oapi.GetImageRequestObject{ - Id: "non-existent", - }) - require.NoError(t, err) - - notFound, ok := resp.(oapi.GetImage404JSONResponse) - require.True(t, ok, "expected 404 response") - assert.Equal(t, "not_found", notFound.Code) - assert.Equal(t, "image not found", notFound.Message) -} - -func TestCreateImage_AsyncWithSSE(t *testing.T) { - svc := newTestService(t) - ctx := ctx() - - // 1. Create image (should return 202 Accepted immediately) - createResp, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ - Body: &oapi.CreateImageRequest{ - Name: "docker.io/library/alpine:latest", - }, - }) - require.NoError(t, err) - - acceptedResp, ok := createResp.(oapi.CreateImage202JSONResponse) - require.True(t, ok, "expected 202 accepted response") - - img := oapi.Image(acceptedResp) - require.Equal(t, "docker.io/library/alpine:latest", img.Name) - require.Equal(t, "img-alpine-latest", img.Id) - require.Contains(t, []oapi.ImageStatus{images.StatusPending, images.StatusPulling}, img.Status) - require.Equal(t, 0, img.Progress) - - // 2. Stream progress via SSE - progressResp, err := svc.GetImageProgress(ctx, oapi.GetImageProgressRequestObject{ - Id: img.Id, - }) - require.NoError(t, err) - - sseResp, ok := progressResp.(oapi.GetImageProgress200TexteventStreamResponse) - if !ok { - t.Fatalf("expected SSE stream, got %T", progressResp) - } - - // Read SSE events - scanner := bufio.NewScanner(sseResp.Body) - lastProgress := 0 - sawPulling := false - sawUnpacking := false - sawConverting := false - - timeout := time.After(3 * time.Minute) - done := make(chan bool) - - go func() { - for scanner.Scan() { - line := scanner.Text() - if !strings.HasPrefix(line, "data: ") { - continue - } - - data := strings.TrimPrefix(line, "data: ") - var update images.ProgressUpdate - if err := json.Unmarshal([]byte(data), &update); err != nil { - continue - } - - t.Logf("SSE: status=%s, progress=%d%%", update.Status, update.Progress) - - // Track which phases we see - if update.Status == images.StatusPulling { - sawPulling = true - } - if update.Status == images.StatusUnpacking { - sawUnpacking = true - } - if update.Status == images.StatusConverting { - sawConverting = true - } - - // Progress should be monotonic - require.GreaterOrEqual(t, update.Progress, lastProgress) - lastProgress = update.Progress - - // Stop when ready - if update.Status == images.StatusReady { - require.Equal(t, 100, update.Progress) - done <- true - return - } - - // Fail on error - if update.Status == images.StatusFailed { - t.Fatalf("Build failed: %v", update.Error) - } - } - }() - - // Wait for completion or timeout - select { - case <-done: - // Success - case <-timeout: - t.Fatal("Build did not complete within 3 minutes") - } - - // Verify we saw at least one intermediate phase (build might be too fast to catch all) - sawAnyPhase := sawPulling || sawUnpacking || sawConverting - require.True(t, sawAnyPhase || lastProgress == 100, "should see at least one build phase or final state") - - // 3. Verify final image state - getResp, err := svc.GetImage(ctx, oapi.GetImageRequestObject{Id: img.Id}) - require.NoError(t, err) - - imgResp, ok := getResp.(oapi.GetImage200JSONResponse) - require.True(t, ok, "expected 200 response") - - finalImg := oapi.Image(imgResp) - require.Equal(t, oapi.ImageStatus(images.StatusReady), finalImg.Status) - require.Equal(t, 100, finalImg.Progress) - require.NotNil(t, finalImg.SizeBytes) - require.Greater(t, *finalImg.SizeBytes, int64(0)) - require.Nil(t, finalImg.QueuePosition) - require.Nil(t, finalImg.Error) -} - diff --git a/lib/images/README.md b/lib/images/README.md index daf63c40..74933239 100644 --- a/lib/images/README.md +++ b/lib/images/README.md @@ -68,7 +68,16 @@ OCI Registry → containers/image → OCI Layout → umoci → rootfs/ → mkfs. Requires `-tags containers_image_openpgp` to avoid C dependency on gpgme. This is a build-time option of the containers/image project to select between gpgme C library with go bindings or the pure Go OpenPGP implementation (slightly slower but doesn't need external system dependency). -## Testing +## Registry Authentication -Integration tests pull alpine:latest (~7MB) and verify full pipeline. No special permissions required. +containers/image automatically uses `~/.docker/config.json` for registry authentication. +```bash +# Login to Docker Hub (avoid rate limits) +docker login + +# Works for any registry +docker login ghcr.io +``` + +No code changes needed - credentials are automatically discovered. \ No newline at end of file diff --git a/lib/images/manager.go b/lib/images/manager.go index 904598a6..4ba6c33c 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -6,20 +6,28 @@ import ( "os" "path/filepath" "regexp" + "sort" "strings" - "sync" "time" "github.com/onkernel/hypeman/lib/oapi" ) +const ( + StatusPending = "pending" + StatusPulling = "pulling" + StatusUnpacking = "unpacking" + StatusConverting = "converting" + StatusReady = "ready" + StatusFailed = "failed" +) + // Manager handles image lifecycle operations type Manager interface { ListImages(ctx context.Context) ([]oapi.Image, error) CreateImage(ctx context.Context, req oapi.CreateImageRequest) (*oapi.Image, error) GetImage(ctx context.Context, id string) (*oapi.Image, error) DeleteImage(ctx context.Context, id string) error - GetProgress(ctx context.Context, id string) (chan ProgressUpdate, error) RecoverInterruptedBuilds() } @@ -27,8 +35,6 @@ type manager struct { dataDir string ociClient *OCIClient queue *BuildQueue - trackers map[string]*ProgressTracker - mu sync.RWMutex } // NewManager creates a new image manager with OCI client @@ -37,9 +43,7 @@ func NewManager(dataDir string, ociClient *OCIClient, maxConcurrentBuilds int) M dataDir: dataDir, ociClient: ociClient, queue: NewBuildQueue(maxConcurrentBuilds), - trackers: make(map[string]*ProgressTracker), } - // Recover interrupted builds on initialization m.RecoverInterruptedBuilds() return m } @@ -59,24 +63,20 @@ func (m *manager) ListImages(ctx context.Context) ([]oapi.Image, error) { } func (m *manager) CreateImage(ctx context.Context, req oapi.CreateImageRequest) (*oapi.Image, error) { - // 1. Generate or validate ID imageID := req.Id if imageID == nil || *imageID == "" { generated := generateImageID(req.Name) imageID = &generated } - // 2. Check if image already exists if imageExists(m.dataDir, *imageID) { return nil, ErrAlreadyExists } - // 3. Create initial metadata with pending status meta := &imageMetadata{ ID: *imageID, Name: req.Name, Status: StatusPending, - Progress: 0, Request: &req, CreatedAt: time.Now(), } @@ -85,40 +85,28 @@ func (m *manager) CreateImage(ctx context.Context, req oapi.CreateImageRequest) return nil, fmt.Errorf("write initial metadata: %w", err) } - // 4. Enqueue the build queuePos := m.queue.Enqueue(*imageID, req, func() { m.buildImage(context.Background(), *imageID, req) }) - meta.QueuePosition = &queuePos - if err := writeMetadata(m.dataDir, *imageID, meta); err != nil { - return nil, fmt.Errorf("update queue position: %w", err) + img := meta.toOAPI() + if queuePos > 0 { + img.QueuePosition = &queuePos } - - // 5. Return immediately (build happens in background) - return meta.toOAPI(), nil + return img, nil } -// buildImage performs the actual image build in the background func (m *manager) buildImage(ctx context.Context, imageID string, req oapi.CreateImageRequest) { - // Create progress tracker - tracker := NewProgressTracker(imageID, m.dataDir) - m.registerTracker(imageID, tracker) - defer tracker.Close() - defer m.unregisterTracker(imageID) defer m.queue.MarkComplete(imageID) - // Use persistent build directory for resumability buildDir := filepath.Join(imageDir(m.dataDir, imageID), ".build") tempDir := filepath.Join(buildDir, "rootfs") - // Ensure build directory exists if err := os.MkdirAll(buildDir, 0755); err != nil { - tracker.Fail(fmt.Errorf("create build dir: %w", err)) + m.updateStatus(imageID, StatusFailed, fmt.Errorf("create build dir: %w", err)) return } - // Cleanup build dir on success defer func() { meta, _ := readMetadata(m.dataDir, imageID) if meta != nil && meta.Status == StatusReady { @@ -126,33 +114,30 @@ func (m *manager) buildImage(ctx context.Context, imageID string, req oapi.Creat } }() - // Phase 1: Pull and unpack (0-90%) - tracker.Update(StatusPulling, 0, nil) - containerMeta, err := m.ociClient.pullAndExportWithProgress(ctx, req.Name, tempDir, tracker) + m.updateStatus(imageID, StatusPulling, nil) + containerMeta, err := m.ociClient.pullAndExport(ctx, req.Name, tempDir) if err != nil { - tracker.Fail(fmt.Errorf("pull and export: %w", err)) + m.updateStatus(imageID, StatusFailed, fmt.Errorf("pull and export: %w", err)) return } - // Phase 2: Convert to ext4 (90-100%) - tracker.Update(StatusConverting, 90, nil) + m.updateStatus(imageID, StatusUnpacking, nil) + + m.updateStatus(imageID, StatusConverting, nil) diskPath := imagePath(m.dataDir, imageID) diskSize, err := convertToExt4(tempDir, diskPath) if err != nil { - tracker.Fail(fmt.Errorf("convert to ext4: %w", err)) + m.updateStatus(imageID, StatusFailed, fmt.Errorf("convert to ext4: %w", err)) return } - // Phase 3: Finalize metadata meta, err := readMetadata(m.dataDir, imageID) if err != nil { - tracker.Fail(fmt.Errorf("read metadata: %w", err)) + m.updateStatus(imageID, StatusFailed, fmt.Errorf("read metadata: %w", err)) return } meta.Status = StatusReady - meta.Progress = 100 - meta.QueuePosition = nil meta.Error = nil meta.SizeBytes = diskSize meta.Entrypoint = containerMeta.Entrypoint @@ -161,79 +146,64 @@ func (m *manager) buildImage(ctx context.Context, imageID string, req oapi.Creat meta.WorkingDir = containerMeta.WorkingDir if err := writeMetadata(m.dataDir, imageID, meta); err != nil { - tracker.Fail(fmt.Errorf("write final metadata: %w", err)) + m.updateStatus(imageID, StatusFailed, fmt.Errorf("write final metadata: %w", err)) return } - - tracker.Complete() } -// GetProgress returns a channel for SSE progress updates -func (m *manager) GetProgress(ctx context.Context, id string) (chan ProgressUpdate, error) { - // Get or create tracker - m.mu.Lock() - tracker, exists := m.trackers[id] - if !exists { - // Check if image exists before creating tracker - if !imageExists(m.dataDir, id) { - m.mu.Unlock() - return nil, ErrNotFound - } - // No active build, create temporary tracker that sends current state - tracker = NewProgressTracker(id, m.dataDir) - m.trackers[id] = tracker +func (m *manager) updateStatus(imageID, status string, err error) { + meta, readErr := readMetadata(m.dataDir, imageID) + if readErr != nil { + return } - m.mu.Unlock() - // Subscribe to progress updates - ch, err := tracker.Subscribe(ctx) + meta.Status = status if err != nil { - return nil, fmt.Errorf("subscribe to progress: %w", err) + errorMsg := err.Error() + meta.Error = &errorMsg } - return ch, nil + writeMetadata(m.dataDir, imageID, meta) } -// RecoverInterruptedBuilds resumes builds that were interrupted by server restart func (m *manager) RecoverInterruptedBuilds() { metas, err := listMetadata(m.dataDir) if err != nil { return // Best effort } + // Sort by created_at to maintain FIFO order + sort.Slice(metas, func(i, j int) bool { + return metas[i].CreatedAt.Before(metas[j].CreatedAt) + }) + for _, meta := range metas { switch meta.Status { case StatusPending, StatusPulling, StatusUnpacking, StatusConverting: - // Re-enqueue the build if meta.Request != nil { - m.queue.Enqueue(meta.ID, *meta.Request, func() { - m.buildImage(context.Background(), meta.ID, *meta.Request) + metaCopy := meta + m.queue.Enqueue(metaCopy.ID, *metaCopy.Request, func() { + m.buildImage(context.Background(), metaCopy.ID, *metaCopy.Request) }) } } } } -// registerTracker adds a tracker to the active map -func (m *manager) registerTracker(imageID string, tracker *ProgressTracker) { - m.mu.Lock() - defer m.mu.Unlock() - m.trackers[imageID] = tracker -} - -// unregisterTracker removes a tracker from the active map -func (m *manager) unregisterTracker(imageID string) { - m.mu.Lock() - defer m.mu.Unlock() - delete(m.trackers, imageID) -} - func (m *manager) GetImage(ctx context.Context, id string) (*oapi.Image, error) { meta, err := readMetadata(m.dataDir, id) if err != nil { return nil, err } - return meta.toOAPI(), nil + + img := meta.toOAPI() + + // Inject live queue position if pending + if meta.Status == StatusPending { + img.QueuePosition = m.queue.GetPosition(id) + } + + return img, nil } func (m *manager) DeleteImage(ctx context.Context, id string) error { diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go index c4d056ad..a369f6c3 100644 --- a/lib/images/manager_test.go +++ b/lib/images/manager_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/onkernel/hypeman/lib/oapi" "github.com/stretchr/testify/require" @@ -22,34 +23,20 @@ func TestCreateImage(t *testing.T) { Name: "docker.io/library/alpine:latest", } - // CreateImage returns immediately with pending status img, err := mgr.CreateImage(ctx, req) require.NoError(t, err) require.NotNil(t, img) require.Equal(t, "docker.io/library/alpine:latest", img.Name) require.Equal(t, "img-alpine-latest", img.Id) - require.Contains(t, []oapi.ImageStatus{StatusPending, StatusPulling}, img.Status) - // Wait for build to complete via progress channel - progressCh, err := mgr.GetProgress(ctx, img.Id) - require.NoError(t, err) - - for update := range progressCh { - if update.Status == StatusReady || update.Status == StatusFailed { - require.Equal(t, StatusReady, update.Status) - break - } - } + waitForReady(t, mgr, ctx, img.Id) - // Verify final state img, err = mgr.GetImage(ctx, img.Id) require.NoError(t, err) require.Equal(t, oapi.ImageStatus(StatusReady), img.Status) - require.Equal(t, 100, img.Progress) require.NotNil(t, img.SizeBytes) require.Greater(t, *img.SizeBytes, int64(0)) - // Verify disk image was created diskPath := filepath.Join(dataDir, "images", img.Id, "rootfs.ext4") _, err = os.Stat(diskPath) require.NoError(t, err) @@ -244,18 +231,33 @@ func TestGenerateImageID(t *testing.T) { // waitForReady waits for an image build to complete func waitForReady(t *testing.T, mgr Manager, ctx context.Context, imageID string) { - progressCh, err := mgr.GetProgress(ctx, imageID) - require.NoError(t, err) + for i := 0; i < 600; i++ { + img, err := mgr.GetImage(ctx, imageID) + if err != nil { + time.Sleep(100 * time.Millisecond) + continue + } - for update := range progressCh { - t.Logf("Progress: status=%s, progress=%d%%", update.Status, update.Progress) - if update.Status == StatusReady || update.Status == StatusFailed { - if update.Status == StatusFailed { - t.Fatalf("Build failed: %v", update.Error) + if i%10 == 0 { + t.Logf("Status: %s", img.Status) + } + + if img.Status == oapi.ImageStatus(StatusReady) { + return + } + + if img.Status == oapi.ImageStatus(StatusFailed) { + errMsg := "" + if img.Error != nil { + errMsg = *img.Error } - break + t.Fatalf("Build failed: %s", errMsg) } + + time.Sleep(100 * time.Millisecond) } + + t.Fatal("Build did not complete within 60 seconds") } diff --git a/lib/images/oci.go b/lib/images/oci.go index 0fd258b7..7d49f8df 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -10,7 +10,6 @@ import ( "github.com/containers/image/v5/docker" "github.com/containers/image/v5/oci/layout" "github.com/containers/image/v5/signature" - "github.com/containers/image/v5/types" "github.com/opencontainers/image-spec/specs-go/v1" rspec "github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/umoci/oci/cas/dir" @@ -36,35 +35,22 @@ func (c *OCIClient) Close() error { return nil } -// pullAndExport pulls an OCI image and exports its rootfs to a directory func (c *OCIClient) pullAndExport(ctx context.Context, imageRef, exportDir string) (*containerMetadata, error) { - return c.pullAndExportWithProgress(ctx, imageRef, exportDir, nil) -} - -// pullAndExportWithProgress pulls an OCI image with progress reporting -func (c *OCIClient) pullAndExportWithProgress(ctx context.Context, imageRef, exportDir string, tracker *ProgressTracker) (*containerMetadata, error) { - // Create temporary OCI layout directory ociLayoutDir := filepath.Join(c.cacheDir, fmt.Sprintf("oci-layout-%d", os.Getpid())) if err := os.MkdirAll(ociLayoutDir, 0755); err != nil { return nil, fmt.Errorf("create oci layout dir: %w", err) } defer os.RemoveAll(ociLayoutDir) - // Pull image to OCI layout with progress - if err := c.pullToOCILayoutWithProgress(ctx, imageRef, ociLayoutDir, tracker); err != nil { + if err := c.pullToOCILayout(ctx, imageRef, ociLayoutDir); err != nil { return nil, fmt.Errorf("pull to oci layout: %w", err) } - // Extract metadata from OCI config meta, err := c.extractOCIMetadata(ociLayoutDir) if err != nil { return nil, fmt.Errorf("extract metadata: %w", err) } - // Unpack layers to export directory (70-90%) - if tracker != nil { - tracker.Update(StatusUnpacking, 70, nil) - } if err := c.unpackLayers(ctx, ociLayoutDir, exportDir); err != nil { return nil, fmt.Errorf("unpack layers: %w", err) } @@ -72,13 +58,7 @@ func (c *OCIClient) pullAndExportWithProgress(ctx context.Context, imageRef, exp return meta, nil } -// pullToOCILayout pulls an image from a registry to an OCI layout directory func (c *OCIClient) pullToOCILayout(ctx context.Context, imageRef, ociLayoutDir string) error { - return c.pullToOCILayoutWithProgress(ctx, imageRef, ociLayoutDir, nil) -} - -// pullToOCILayoutWithProgress pulls with progress reporting (0-70%) -func (c *OCIClient) pullToOCILayoutWithProgress(ctx context.Context, imageRef, ociLayoutDir string, tracker *ProgressTracker) error { // Parse source reference (docker://...) srcRef, err := docker.ParseReference("//" + imageRef) if err != nil { @@ -100,36 +80,8 @@ func (c *OCIClient) pullToOCILayoutWithProgress(ctx context.Context, imageRef, o } defer policyContext.Destroy() - // Setup progress reporting if tracker provided - var progressChan chan types.ProgressProperties - if tracker != nil { - progressChan = make(chan types.ProgressProperties) - go func() { - var totalBytes int64 - var downloadedBytes int64 - - for p := range progressChan { - // Track total and downloaded bytes - if p.Event == types.ProgressEventNewArtifact { - totalBytes += p.Artifact.Size - } - if p.Event == types.ProgressEventRead { - downloadedBytes = int64(p.Offset) - } - - // Calculate percentage (0-70% of total progress) - if totalBytes > 0 { - pct := int((float64(downloadedBytes) / float64(totalBytes)) * 70) - tracker.Update(StatusPulling, pct, nil) - } - } - }() - } - - // Pull image _, err = copy.Image(ctx, policyContext, destRef, srcRef, ©.Options{ ReportWriter: os.Stdout, - Progress: progressChan, }) if err != nil { return fmt.Errorf("copy image: %w", err) diff --git a/lib/images/progress.go b/lib/images/progress.go deleted file mode 100644 index c6b6a38b..00000000 --- a/lib/images/progress.go +++ /dev/null @@ -1,225 +0,0 @@ -package images - -import ( - "context" - "encoding/json" - "fmt" - "io" - "sync" -) - -// Build status constants -const ( - StatusPending = "pending" - StatusPulling = "pulling" - StatusUnpacking = "unpacking" - StatusConverting = "converting" - StatusReady = "ready" - StatusFailed = "failed" -) - -// ProgressUpdate represents a status update during image build -type ProgressUpdate struct { - Status string `json:"status"` - Progress int `json:"progress"` - QueuePosition *int `json:"queue_position,omitempty"` - Error *string `json:"error,omitempty"` -} - -// ProgressTracker tracks build progress and broadcasts updates to SSE subscribers -type ProgressTracker struct { - imageID string - dataDir string - subscribers []chan ProgressUpdate - mu sync.RWMutex - closed bool -} - -// NewProgressTracker creates a new progress tracker -func NewProgressTracker(imageID, dataDir string) *ProgressTracker { - return &ProgressTracker{ - imageID: imageID, - dataDir: dataDir, - subscribers: make([]chan ProgressUpdate, 0), - } -} - -// Update updates the progress and broadcasts to all subscribers -func (p *ProgressTracker) Update(status string, progress int, queuePos *int) { - p.mu.RLock() - defer p.mu.RUnlock() - - if p.closed { - return - } - - // Update metadata on disk - meta, err := readMetadata(p.dataDir, p.imageID) - if err != nil { - return // Best effort - } - - meta.Status = status - meta.Progress = progress - meta.QueuePosition = queuePos - writeMetadata(p.dataDir, p.imageID, meta) - - // Broadcast to subscribers - update := ProgressUpdate{ - Status: status, - Progress: progress, - QueuePosition: queuePos, - } - - for _, ch := range p.subscribers { - select { - case ch <- update: - default: - // Non-blocking send (skip slow consumers) - } - } -} - -// Fail marks the build as failed with error message -func (p *ProgressTracker) Fail(err error) { - p.mu.RLock() - defer p.mu.RUnlock() - - if p.closed { - return - } - - meta, metaErr := readMetadata(p.dataDir, p.imageID) - if metaErr != nil { - return - } - - meta.Status = StatusFailed - meta.Progress = 0 - meta.QueuePosition = nil - errorMsg := err.Error() - meta.Error = &errorMsg - writeMetadata(p.dataDir, p.imageID, meta) - - // Broadcast failure - update := ProgressUpdate{ - Status: StatusFailed, - Error: &errorMsg, - } - - for _, ch := range p.subscribers { - select { - case ch <- update: - default: - } - } -} - -// Complete marks the build as complete -func (p *ProgressTracker) Complete() { - p.Update(StatusReady, 100, nil) -} - -// Subscribe adds a new SSE subscriber and returns their channel -func (p *ProgressTracker) Subscribe(ctx context.Context) (chan ProgressUpdate, error) { - p.mu.Lock() - defer p.mu.Unlock() - - if p.closed { - return nil, fmt.Errorf("tracker closed") - } - - ch := make(chan ProgressUpdate, 10) // Buffered for slow consumers - p.subscribers = append(p.subscribers, ch) - - // Send current state immediately - meta, err := readMetadata(p.dataDir, p.imageID) - if err == nil { - update := ProgressUpdate{ - Status: meta.Status, - Progress: meta.Progress, - QueuePosition: meta.QueuePosition, - Error: meta.Error, - } - ch <- update - } - - // Close channel when context is done - go func() { - <-ctx.Done() - p.Unsubscribe(ch) - }() - - return ch, nil -} - -// Unsubscribe removes a subscriber -func (p *ProgressTracker) Unsubscribe(ch chan ProgressUpdate) { - p.mu.Lock() - defer p.mu.Unlock() - - for i, sub := range p.subscribers { - if sub == ch { - p.subscribers = append(p.subscribers[:i], p.subscribers[i+1:]...) - close(ch) - break - } - } -} - -// Close closes all subscriber channels -func (p *ProgressTracker) Close() { - p.mu.Lock() - defer p.mu.Unlock() - - if p.closed { - return - } - - p.closed = true - for _, ch := range p.subscribers { - close(ch) - } - p.subscribers = nil -} - -// ToSSEReader converts a progress channel to an io.ReadCloser for SSE streaming -func ToSSEReader(ch chan ProgressUpdate) io.ReadCloser { - return &sseStream{ch: ch} -} - -// sseStream implements io.ReadCloser for SSE streaming -type sseStream struct { - ch chan ProgressUpdate - buffer []byte -} - -func (s *sseStream) Read(p []byte) (n int, err error) { - // If we have buffered data, return it first - if len(s.buffer) > 0 { - n = copy(p, s.buffer) - s.buffer = s.buffer[n:] - return n, nil - } - - // Get next update from channel - update, ok := <-s.ch - if !ok { - return 0, io.EOF - } - - // Format as SSE - data, _ := json.Marshal(update) - msg := fmt.Sprintf("data: %s\n\n", data) - s.buffer = []byte(msg) - - // Copy to output buffer - n = copy(p, s.buffer) - s.buffer = s.buffer[n:] - return n, nil -} - -func (s *sseStream) Close() error { - return nil -} - diff --git a/lib/images/storage.go b/lib/images/storage.go index 9db8a3fc..3694bdd8 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -10,36 +10,29 @@ import ( "github.com/onkernel/hypeman/lib/oapi" ) -// imageMetadata represents the metadata stored on disk type imageMetadata struct { - ID string `json:"id"` - Name string `json:"name"` - Status string `json:"status"` - Progress int `json:"progress"` - QueuePosition *int `json:"queue_position,omitempty"` - Error *string `json:"error,omitempty"` - Request *oapi.CreateImageRequest `json:"request,omitempty"` - SizeBytes int64 `json:"size_bytes"` - Entrypoint []string `json:"entrypoint,omitempty"` - Cmd []string `json:"cmd,omitempty"` - Env map[string]string `json:"env,omitempty"` - WorkingDir string `json:"working_dir,omitempty"` - CreatedAt time.Time `json:"created_at"` + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Error *string `json:"error,omitempty"` + Request *oapi.CreateImageRequest `json:"request,omitempty"` + SizeBytes int64 `json:"size_bytes"` + Entrypoint []string `json:"entrypoint,omitempty"` + Cmd []string `json:"cmd,omitempty"` + Env map[string]string `json:"env,omitempty"` + WorkingDir string `json:"working_dir,omitempty"` + CreatedAt time.Time `json:"created_at"` } -// toOAPI converts internal metadata to OpenAPI schema func (m *imageMetadata) toOAPI() *oapi.Image { img := &oapi.Image{ - Id: m.ID, - Name: m.Name, - Status: oapi.ImageStatus(m.Status), - Progress: m.Progress, - QueuePosition: m.QueuePosition, - Error: m.Error, - CreatedAt: m.CreatedAt, + Id: m.ID, + Name: m.Name, + Status: oapi.ImageStatus(m.Status), + Error: m.Error, + CreatedAt: m.CreatedAt, } - // Only set size_bytes when ready if m.Status == StatusReady && m.SizeBytes > 0 { sizeBytes := m.SizeBytes img.SizeBytes = &sizeBytes @@ -61,22 +54,18 @@ func (m *imageMetadata) toOAPI() *oapi.Image { return img } -// imageDir returns the directory path for an image func imageDir(dataDir, imageID string) string { return filepath.Join(dataDir, "images", imageID) } -// imagePath returns the path to the rootfs disk image func imagePath(dataDir, imageID string) string { return filepath.Join(imageDir(dataDir, imageID), "rootfs.ext4") } -// metadataPath returns the path to the metadata file func metadataPath(dataDir, imageID string) string { return filepath.Join(imageDir(dataDir, imageID), "metadata.json") } -// writeMetadata writes metadata atomically using temp file + rename func writeMetadata(dataDir, imageID string, meta *imageMetadata) error { dir := imageDir(dataDir, imageID) if err := os.MkdirAll(dir, 0755); err != nil { @@ -88,23 +77,20 @@ func writeMetadata(dataDir, imageID string, meta *imageMetadata) error { return fmt.Errorf("marshal metadata: %w", err) } - // Write to temp file first tempPath := metadataPath(dataDir, imageID) + ".tmp" if err := os.WriteFile(tempPath, data, 0644); err != nil { return fmt.Errorf("write temp metadata: %w", err) } - // Atomic rename finalPath := metadataPath(dataDir, imageID) if err := os.Rename(tempPath, finalPath); err != nil { - os.Remove(tempPath) // cleanup + os.Remove(tempPath) return fmt.Errorf("rename metadata: %w", err) } return nil } -// readMetadata reads metadata from disk func readMetadata(dataDir, imageID string) (*imageMetadata, error) { path := metadataPath(dataDir, imageID) data, err := os.ReadFile(path) @@ -120,19 +106,19 @@ func readMetadata(dataDir, imageID string) (*imageMetadata, error) { return nil, fmt.Errorf("unmarshal metadata: %w", err) } - // Validate that disk image exists - diskPath := imagePath(dataDir, imageID) - if _, err := os.Stat(diskPath); err != nil { - if os.IsNotExist(err) { - return nil, fmt.Errorf("disk image missing: %s", diskPath) + if meta.Status == StatusReady { + diskPath := imagePath(dataDir, imageID) + if _, err := os.Stat(diskPath); err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("disk image missing: %s", diskPath) + } + return nil, fmt.Errorf("stat disk image: %w", err) } - return nil, fmt.Errorf("stat disk image: %w", err) } return &meta, nil } -// listMetadata lists all image metadata by scanning the images directory func listMetadata(dataDir string) ([]*imageMetadata, error) { imagesDir := filepath.Join(dataDir, "images") entries, err := os.ReadDir(imagesDir) @@ -151,7 +137,6 @@ func listMetadata(dataDir string) ([]*imageMetadata, error) { meta, err := readMetadata(dataDir, entry.Name()) if err != nil { - // Skip invalid entries, log but don't fail continue } metas = append(metas, meta) @@ -160,13 +145,11 @@ func listMetadata(dataDir string) ([]*imageMetadata, error) { return metas, nil } -// imageExists checks if an image already exists func imageExists(dataDir, imageID string) bool { _, err := readMetadata(dataDir, imageID) return err == nil } -// deleteImage removes the entire image directory func deleteImage(dataDir, imageID string) error { dir := imageDir(dataDir, imageID) if _, err := os.Stat(dir); err != nil { @@ -182,4 +165,3 @@ func deleteImage(dataDir, imageID string) error { return nil } - diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 84acbdf5..a941e891 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -174,9 +174,6 @@ type Image struct { // Name OCI image reference Name string `json:"name"` - // Progress Build progress percentage - Progress int `json:"progress"` - // QueuePosition Position in build queue (null if not queued) QueuePosition *int `json:"queue_position"` @@ -422,9 +419,6 @@ type ClientInterface interface { // GetImage request GetImage(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) - // GetImageProgress request - GetImageProgress(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) - // ListInstances request ListInstances(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -543,18 +537,6 @@ func (c *Client) GetImage(ctx context.Context, id string, reqEditors ...RequestE return c.Client.Do(req) } -func (c *Client) GetImageProgress(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewGetImageProgressRequest(c.Server, id) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - func (c *Client) ListInstances(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewListInstancesRequest(c.Server) if err != nil { @@ -909,40 +891,6 @@ func NewGetImageRequest(server string, id string) (*http.Request, error) { return req, nil } -// NewGetImageProgressRequest generates requests for GetImageProgress -func NewGetImageProgressRequest(server string, id string) (*http.Request, error) { - var err error - - var pathParam0 string - - pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) - if err != nil { - return nil, err - } - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/images/%s/progress", pathParam0) - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - req, err := http.NewRequest("GET", queryURL.String(), nil) - if err != nil { - return nil, err - } - - return req, nil -} - // NewListInstancesRequest generates requests for ListInstances func NewListInstancesRequest(server string) (*http.Request, error) { var err error @@ -1508,9 +1456,6 @@ type ClientWithResponsesInterface interface { // GetImageWithResponse request GetImageWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetImageResponse, error) - // GetImageProgressWithResponse request - GetImageProgressWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetImageProgressResponse, error) - // ListInstancesWithResponse request ListInstancesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*ListInstancesResponse, error) @@ -1675,29 +1620,6 @@ func (r GetImageResponse) StatusCode() int { return 0 } -type GetImageProgressResponse struct { - Body []byte - HTTPResponse *http.Response - JSON404 *Error - JSON500 *Error -} - -// Status returns HTTPResponse.Status -func (r GetImageProgressResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r GetImageProgressResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - type ListInstancesResponse struct { Body []byte HTTPResponse *http.Response @@ -2066,15 +1988,6 @@ func (c *ClientWithResponses) GetImageWithResponse(ctx context.Context, id strin return ParseGetImageResponse(rsp) } -// GetImageProgressWithResponse request returning *GetImageProgressResponse -func (c *ClientWithResponses) GetImageProgressWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetImageProgressResponse, error) { - rsp, err := c.GetImageProgress(ctx, id, reqEditors...) - if err != nil { - return nil, err - } - return ParseGetImageProgressResponse(rsp) -} - // ListInstancesWithResponse request returning *ListInstancesResponse func (c *ClientWithResponses) ListInstancesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*ListInstancesResponse, error) { rsp, err := c.ListInstances(ctx, reqEditors...) @@ -2402,39 +2315,6 @@ func ParseGetImageResponse(rsp *http.Response) (*GetImageResponse, error) { return response, nil } -// ParseGetImageProgressResponse parses an HTTP response from a GetImageProgressWithResponse call -func ParseGetImageProgressResponse(rsp *http.Response) (*GetImageProgressResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &GetImageProgressResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON404 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON500 = &dest - - } - - return response, nil -} - // ParseListInstancesResponse parses an HTTP response from a ListInstancesWithResponse call func ParseListInstancesResponse(rsp *http.Response) (*ListInstancesResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -2993,9 +2873,6 @@ type ServerInterface interface { // Get image details // (GET /images/{id}) GetImage(w http.ResponseWriter, r *http.Request, id string) - // Stream image build progress (SSE) - // (GET /images/{id}/progress) - GetImageProgress(w http.ResponseWriter, r *http.Request, id string) // List instances // (GET /instances) ListInstances(w http.ResponseWriter, r *http.Request) @@ -3071,12 +2948,6 @@ func (_ Unimplemented) GetImage(w http.ResponseWriter, r *http.Request, id strin w.WriteHeader(http.StatusNotImplemented) } -// Stream image build progress (SSE) -// (GET /images/{id}/progress) -func (_ Unimplemented) GetImageProgress(w http.ResponseWriter, r *http.Request, id string) { - w.WriteHeader(http.StatusNotImplemented) -} - // List instances // (GET /instances) func (_ Unimplemented) ListInstances(w http.ResponseWriter, r *http.Request) { @@ -3280,37 +3151,6 @@ func (siw *ServerInterfaceWrapper) GetImage(w http.ResponseWriter, r *http.Reque handler.ServeHTTP(w, r) } -// GetImageProgress operation middleware -func (siw *ServerInterfaceWrapper) GetImageProgress(w http.ResponseWriter, r *http.Request) { - - var err error - - // ------------- Path parameter "id" ------------- - var id string - - err = runtime.BindStyledParameterWithOptions("simple", "id", chi.URLParam(r, "id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) - if err != nil { - siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) - return - } - - ctx := r.Context() - - ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) - - r = r.WithContext(ctx) - - handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - siw.Handler.GetImageProgress(w, r, id) - })) - - for _, middleware := range siw.HandlerMiddlewares { - handler = middleware(handler) - } - - handler.ServeHTTP(w, r) -} - // ListInstances operation middleware func (siw *ServerInterfaceWrapper) ListInstances(w http.ResponseWriter, r *http.Request) { @@ -3835,9 +3675,6 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/images/{id}", wrapper.GetImage) }) - r.Group(func(r chi.Router) { - r.Get(options.BaseURL+"/images/{id}/progress", wrapper.GetImageProgress) - }) r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/instances", wrapper.ListInstances) }) @@ -4044,51 +3881,6 @@ func (response GetImage500JSONResponse) VisitGetImageResponse(w http.ResponseWri return json.NewEncoder(w).Encode(response) } -type GetImageProgressRequestObject struct { - Id string `json:"id"` -} - -type GetImageProgressResponseObject interface { - VisitGetImageProgressResponse(w http.ResponseWriter) error -} - -type GetImageProgress200TexteventStreamResponse struct { - Body io.Reader - ContentLength int64 -} - -func (response GetImageProgress200TexteventStreamResponse) VisitGetImageProgressResponse(w http.ResponseWriter) error { - w.Header().Set("Content-Type", "text/event-stream") - if response.ContentLength != 0 { - w.Header().Set("Content-Length", fmt.Sprint(response.ContentLength)) - } - w.WriteHeader(200) - - if closer, ok := response.Body.(io.ReadCloser); ok { - defer closer.Close() - } - _, err := io.Copy(w, response.Body) - return err -} - -type GetImageProgress404JSONResponse Error - -func (response GetImageProgress404JSONResponse) VisitGetImageProgressResponse(w http.ResponseWriter) error { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(404) - - return json.NewEncoder(w).Encode(response) -} - -type GetImageProgress500JSONResponse Error - -func (response GetImageProgress500JSONResponse) VisitGetImageProgressResponse(w http.ResponseWriter) error { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(500) - - return json.NewEncoder(w).Encode(response) -} - type ListInstancesRequestObject struct { } @@ -4635,9 +4427,6 @@ type StrictServerInterface interface { // Get image details // (GET /images/{id}) GetImage(ctx context.Context, request GetImageRequestObject) (GetImageResponseObject, error) - // Stream image build progress (SSE) - // (GET /images/{id}/progress) - GetImageProgress(ctx context.Context, request GetImageProgressRequestObject) (GetImageProgressResponseObject, error) // List instances // (GET /instances) ListInstances(ctx context.Context, request ListInstancesRequestObject) (ListInstancesResponseObject, error) @@ -4839,32 +4628,6 @@ func (sh *strictHandler) GetImage(w http.ResponseWriter, r *http.Request, id str } } -// GetImageProgress operation middleware -func (sh *strictHandler) GetImageProgress(w http.ResponseWriter, r *http.Request, id string) { - var request GetImageProgressRequestObject - - request.Id = id - - handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { - return sh.ssi.GetImageProgress(ctx, request.(GetImageProgressRequestObject)) - } - for _, middleware := range sh.middlewares { - handler = middleware(handler, "GetImageProgress") - } - - response, err := handler(r.Context(), w, r, request) - - if err != nil { - sh.options.ResponseErrorHandlerFunc(w, r, err) - } else if validResponse, ok := response.(GetImageProgressResponseObject); ok { - if err := validResponse.VisitGetImageProgressResponse(w); err != nil { - sh.options.ResponseErrorHandlerFunc(w, r, err) - } - } else if response != nil { - sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) - } -} - // ListInstances operation middleware func (sh *strictHandler) ListInstances(w http.ResponseWriter, r *http.Request) { var request ListInstancesRequestObject @@ -5222,61 +4985,59 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xce28TuRb/Kpbv/aNISfNoYSH3LyjsUolCRdmudFkUOeOTxFuPPbU9gYD63a/8mMlM", - "xnkU2izZWwmJTMY+xz7ndx4+x+k3nMg0kwKE0XjwDetkCilxH58bQ5LppeR5Cu/hOgdt7NeZkhkow8AN", + "H4sIAAAAAAAC/+xce28TuRb/Kpbv/aNISfNoYSH3LyjsUmkLFYWudFkUOeOTxFuPPbU9gYD63a/8mMlM", + "xnkU2izZWwmJTMY+xz7ndx4+x+k3nMg0kwKE0XjwDetkCilxH58bQ5LppeR5Cu/gOgdt7NeZkhkow8AN", "SmUuzDAjZmqfKOhEscwwKfAAnxMzRZ+noADNHBWkpzLnFI0AuXlAcQvDF5JmHPAAd1JhOpQYglvYzDP7", "lTaKiQm+aWEFhErB557NmOTc4MGYcA2tJbZnljQiGtkpbTenpDeSkgMR+MZRvM6ZAooHH6vb+FQOlqO/", - "IDGW+YkCYuA0JZPVkmC0KYF37gPhKMm1kSliFIRhYwYKHZDcyPYEBChigCI2RkIalCk5YxToo5pkWDpp", + "IDGW+YkCYuA0JZPVkmC0KYG37gPhKMm1kSliFIRhYwYKHZDcyPYEBChigCI2RkIalCk5YxToo5pkWDpp", "iwkTX9qzXkw4gqQQ4X5yiphdM1IwBgUiAXQAh5PDFqIyuQJ1yGSHs5Eiat5x5AecGNCmznz92OZylkTr", - "1rZGqEIbIpLVcgUxs/8RSpkX5nntdUMWdRm8EjOmpEhBGDQjipERB13d3jf89t3LV8NXby/xwHKmeeKm", - "tvD5u/cf8AAfdbtdS7ex/pjCfxfsOoeqnsdSITMFxMI+0UGhYjSao4RwDmpJ2UKbNhklvf5RTNdOo03O", - "DpwVxg38JFMlU1gBoBRSqebDlHwZpqOaiR13nz1pWBj5wtI8RX4W+szMFE2lyXg+QUygsxdV5p5A4MiE", - "gQmoKss6u163f7zM7gXRUPBqkO93j5/GyMdN4nWeEtG2jsECAblBVUGl8/Znqa64JLQdFVQmlRmmJMuY", - "mOiIy5PKoOI1GiuZoqnUBhmJJrm3FmYgdTP/rWCMB/hfnYUH7gT327F0zjyZCvaIUmTunlkKMjdDDYkU", - "VNckePSk212W4Ac/3oFRJ4RD28j2V1ASaUiJMCyp2cQvfUuiKdNZkuV1Zv1lTm/zdAQKyTGaMWVywtHJ", - "+e814v0oZRcfIgL14UdbARIXj7aVoJ/oY5i1/qYYl/wUs8EoAMLb2GqntSEo3mcomEnetjGyfYtA4Jcb", - "Rbsj5aUfo6fZVxhORk2SF+yr9WlowiZkNDd1n9qLoCcWFRb0Y6J+pZRUTeEmkka2+DzLOEuIfWrrDBI2", - "ZgkCSwHZCeggJcmUCShtvy7VEaFDFdTZisUUQxiPwPN5GZUCszASHVhTS3NuWMbBv9OPtsWu2/lLRylm", - "/UwIUEMoxHMLSiloHY0eS36x2Es5xHkOCqN8MrEiqYrujGnNxAQV2kVjBpwOfOaxMTtw2lwsbCUOwh62", - "RMMb+RlUm8MMeBUE3qLsYlOpAJU48UpbCsIzwhkdMpHlJh4xV4jy11yZKRRIQGRkHa/NALzCqkx8zLa2", - "Ppa5oFFhNcTxGgj32XZdEtoQk4eMKU+tbOWVleeCnbzaqI5AJKaG0yLvWFJAGnF2J2cvffBLpDCECVAo", - "BUNCbl+u6CN2mSRu4bbFFCWQSoHkePwfu4LSVJpeLufc4hQPjMqhaSCJc9J0SExkafadRbSNodqQNEMH", - "7389OTo6elZ3Cf1u/3G722v3Hn/odQdd+++/uIXHUqWWLqbEQNsSiaEDhFHzTDIRWcGr8t12Mur4BLy9", - "oHmopz8moHvIqbfZyzd8/vzDa3vSy7XqcJkQ3tEjJgaV5/Jx8cJ98I8jJqK5eOkMl1bqbD+Yqo2rHt+I", - "aTQmjC+dP7Oc8/D9wO5EQFIiRTovsEKuleR8mzPB/RztfujM1rJWPVGgIzHuRc44RcV7lIFKQBjv3Csh", - "v9vCqT8XFE9M+KdoLnmdQw7DTGrm2TQzaf/GJhkjtwI3Ax1YHRQpkvuqniD1V2qpwtylHT5taTB+yfQV", - "0iG9cWMCz1wYxl1RYV7j+PjoydNfus96/YpzYMI8OcZbLaV02zGph7et0qdnIKiPwBat/lMuMpJc+c+J", - "FDNryO7BrdX6LI/1WjAo3jWAMAOloxrx0cqQCbK5AJt4IC0oltjaaCX2iMXEZEhZxGb/8C8RZQoSY898", - "W3gW3CFZtpn1mpS/lHRpCLU4Eo2I4UgfCYp3H4CObhuA7qNq0hDB+JqKWArE+Rxd54Rbf0cRlSlhonn+", - "qFQ6Dp333wY8U6KHWpBMT2VEun9MwWVfBBVjEHxh2uhQiGG6rMRUlxIKictVwu9y6D9H+aYGOSnGbJLb", - "5Dytl26+t1qzgvpoTys1d1SWyRSbEQNDlkX4+Xfo9BwRSoODWWyn96x/2Hvy9LDX7R72utvYgTZErfIx", - "F/bddziYxysdzDbLMbBJfoXHvHCD3SyZZSs3IbNb7aG/wUlu3EO0rBaroyUB8iSUim9TObvPapkvdwFF", - "xYjdFcsKBGwdNS8KwCw5wqJO7sgN/hRt5GtudIAuz85QoI5GuXEpYDADdHDCZU7R63kGasa0VEgQw2bw", - "yFJ4nwvBxMRSsLk/SewbPkfKf79+8jnJtedu52buaf2Mi2luqPws3Bw9zQ2yT27JdgshIK0n4Q1jgN5K", - "NyestIWEXI5sfjgRdDRvDl+OggcJEWhkjw3aSAX00Z+ikl0GSeMWDhLDLey3j1u42JX96FfnPjnGFU0v", - "zKnqLRspkiuHD62TXuGbmXBVEzcOXZ5VjeJp1Mamcj1B6QnaYXVicXKZkkYmktfq3dgkWUVe/imnWWT/", - "SxazWF2ruveYhXhrbIqMBOseGrnGbk5f2tNRMXZNarLRHd51Ftt9dtsyyu2zr/Xl8XXdat82tu9Wyq/a", - "oN4ovb0pxddOQYHJRi/eiBg/eDOA6eJKQM3w7+N+QHFEaHJed2GgiLrDGCaDVtdgctWBYEkXCx6t9XcS", - "LCAgyRUz8wsbxL3MR0AUqOe5l7mL7m4T7usF86kxGb65ca2EccSX/AYCFEvQ8/NTd2xKiSATGycvzxBn", - "Y0jmCQeUu7J/I4i5bvC7k9O2PQxQVCTp7vjIjBOIHZ0SYenjSsEBdw97h67XLjMQJGN4gI/cVy1sxeC2", - "2JmW9e8JONhZ0DlfdErd2k2okFvJ6kwK7WXT73Z9w0CYgFey6Bl1/tK+5OEzok35UuDgRLhkjFYMiUOV", - "X6jPnXSepkTN7d7dtyiZQnLlXnVc/qRXbugN0+bUD/nBHW2VCvoyfzP/a+zUrstmrmH5Ny183O3dmYR9", - "8y/C9ndBcjOVin0Fapk+vkO1rmR6KgwoQTjSoGagQiunaoR48LFufh8/3Xyq6t2JayGrTOqIriv3jLB3", - "DKDNC0nnd7bFyE2mm7oTsuHspoG0/p2tIAAsImRXAhkVdU+f1RM9F8kjj64dKPoFoajoAz8gej2iz3PO", - "EREUhZozKpsSVb/W+cbojQ8xHPyBr475l+77AvMZUSQFA0o7/syu1YXAIqHy6Uodrq2KNJbj66cGlI9X", - "Vd/8CqlX/PEOdLDU/t0j3XulFdpurQzFO1Rrd1ceqrgM8gCTjTD5DULMWwhtyTN0qn3HtSg6X/Rl/gY0", - "GfhiOjADYdraKCBpXcDLBBvCLFaPHA0UaDxAaCOELpykAopG9Vb0wcXFq0cBUqH8sSGPLkftJJUuGk23", - "yabLFT6kH9sk1FVxrc2pF02/e0yrl+6yb5VZ352KF3iLCTyUB0Nd5yGj/gkh7VHkcmp3AFq0qus+btuc", - "eoH5vymtLkC388y6YLynIU9mDgQ0ZNmVOLIyRdqprru79Vk7z7j3Gj4u6W6IrulAOlxO1qfdYfgbObmX", - "rLvVuDckOZefkV0XOvApsi8nuzyv5Vle56DmC55jNwdX+SyX/5u//Vvdh+dM+F+bKDC5Ev7uGbi72THu", - "4d54hHcvdhHg7g8aLT8h44SJWx5J3sjJzs8he+6X/VGk2ITHaeQM4s0rtNhdZyyamb73A/7Rrru4Z/A3", - "Q+y4++z+WZ9IMeYsMai9wIhdBRM2nRN0NEdSVS9w7BP4A1gXO3OeMewriv/i3Ur8h7sj/2j8L3T/f24B", - "iVQKEuOvde1Xm6OSTlVM+cDdBFvcsGoV6frl2Vk8IIRLeZ1v/sPppjPc4s8v3FP2FSFSLG0vrCxcu6AQ", - "7urs3MJkeYtkT7s4VnDFFpxDr5414167+mdB9gGXd1/si/1hlK1KfTu1ivIG289iFbuOQGENhLufOtXk", - "sS8G6pFW7MTIpYJg5R74ypbHZXkT/P4bHsEp3KLdUezgoTK8RbOjIqx1rY7SNd9fo+M7fN/dKbdA2UrP", - "99Di+OlbHLNChwsvtmVT4/4Sj61aGmXKuduGxuXPE0+Z3stQGm4szcoQtarqvUuAdXfnFHfdQ7nc43PR", - "b1AE20r/xBGwFD0clmvpCeGIwgy4zNzPpv1Y3MK54uHK/aDj/8bFVGrjfnaEbz7d/C8AAP//4IBc+4FR", - "AAA=", + "1rZGqEIbIpLVcgUxs/8RSpkX5nntdUMWdRm8EjOmpEhBGDQjipERB13d3jf85u3LV8NXby7xwHKmeeKm", + "tvD523fv8QAfdbtdS7ex/pjCPwh2nUNVz2OpkJkCYmGf6KBQMRrNUUI4B7WkbKFNm4ySXv8opmun0SZn", + "B84K4wZ+kqmSKawAUAqpVPNhSr4M01HNxI67z540LIx8YWmeIj8LfWZmiqbSZDyfICbQ2Ysqc08gcGTC", + "wARUlWWdXa/bP15m94JoKHg1yPe7x09j5OMm8TpPiWhbx2CBgNygqqDSefuzVFdcEtqOCiqTygxTkmVM", + "THTE5UllUPEajZVM0VRqg4xEk9xbCzOQupn/VjDGA/yvzsIDd4L77Vg6Z55MBXtEKTJ3zywFmZuhhkQK", + "qmsSPHrS7S5L8L0f78CoE8KhbWT7KyiJNKREGJbUbOKXviXRlOksyfI6s/4ypzd5OgKF5BjNmDI54ejk", + "/EONeD9K2cWHiEB9+NFWgMTFo20l6Cf6GGatvynGJT/FbDAKgPA2ttppbQiK9xkKZpK3bYxs3yIQ+OVG", + "0e5IeenH6Gn2FYaTUZPkBftqfRqasAkZzU3dp/Yi6IlFhQX9mKhfKSVVU7iJpJEtPs8yzhJin9o6g4SN", + "WYLAUkB2AjpISTJlAkrbr0t1ROhQBXW2YjHFEMYj8HxeRqXALIxEB9bU0pwblnHw7/SjbbHrdv7SUYpZ", + "PxMC1BAK8dyCUgpaR6PHkl8s9lIOcZ6DwiifTKxIqqI7Y1ozMUGFdtGYAacDn3lszA6cNhcLW4mDsIct", + "0fC7/AyqzWEGvAoCb1F2salUgEqceKUtBeEZ4YwOmchyE4+YK0T5a67MFAokIDKyjtdmAF5hVSY+Zltb", + "H8tc0KiwGuJ4DYT7bLsuCW2IyUPGlKdWtvLKynPBTl5tVEcgElPDaZF3LCkgjTi7k7OXPvglUhjCBCiU", + "giEhty9X9BG7TBK3cNtiihJIpUByPP6PXUFpKk0vl3NucYoHRuXQNJDEOWk6JCayNPvOItrGUG1ImqGD", + "d7+eHB0dPau7hH63/7jd7bV7j9/3uoOu/fdf3MJjqVJLF1NioG2JxNABwqh5JpmIrOBV+W47GXV8At5e", + "0DzU0x8T0D3k1Nvs5Rs+f/7+tT3p5Vp1uEwI7+gRE4PKc/m4eOE++McRE9FcvHSGSyt1th9M1cZVj2/E", + "NBoTxpfOn1nOefh+YHciICmRIp0XWCHXSnK+zZngfo52P3Rma+HrHHIYZlIzz6KZ2fo3NuiPcsYpcjPQ", + "gZVJkbK4r+oJS3+l1Crpn0sDfBrRYPyS6SukQ7rhxgSeuTCMu0P+vMbx8dGTp790n/X6FWNlwjw5xlst", + "pXSjS4cQt+fwtlX62AwE9RHRosd/ykVGkiv/OZFiZg3LPbi1Wh/isVdzzsW7hmJmoHRUIz56GDJBNjaz", + "iVfsgmKp642otUceJiZDyiI29Id/iShTkBh7BtvC0nGHZNlm1mtS8FLSFWceDUvhXB2JTHcfBY5uGwXu", + "o3TREMH4mopYHsL5HF3nhFunQxGVKWGieQiolBsOnQveBjFToodakExPZUS6f0zBpUAEFWMQfGHa6FAN", + "Ybosh1SXEqp5y6W67/KqP0cNpQY5KcZsktsMOa3XT763ZLKC+mhPyyV3VBvJFJsRA0OWRfj5d+j0HBFK", + "FejasRX3nvUPe0+eHva63cNedxs70IaoVT7mwr77DgfzeKWD2WY5BjbJr/CYF26wmyWzbOUmZHarPfQ3", + "OMmNe4jWtmLFrCRAnoR67W3KV/dZsvI1J6CoGLG7ilWBgK2j5kUBmCVHWBSrHbnBn6KNfOGLDtDl2RkK", + "1NEoNy7vC2aADk64zCl6Pc9AzZiWCgli2AweWQrvciGYmFgKNgEniX3D50j579dPPie59tzt3Mw9rZ9x", + "Mc0NlZ+Fm6OnuUH2yS3ZbiEEpPUkvGEM0Bvp5oSVtpCQy5HNDyeCjubN4ctR8CAhAo1s7q6NVEAf/Skq", + "KWWQNG7hIDHcwn77uIWLXdmPfnXuk2Nc0fTCnKrespEiuZr00DrpFb6ZCVe6cOPQ5VnVKJ5GbWwq1xOU", + "nqAdVicWJ5cpaWQiea3ojE2SVeTln3KaRfa/ZDGL1bWqe49ZiLfGpshIsO6hkWvs5vSlPRIVY9ekJhvd", + "4V1nsd1nt61l3D77Wl+jXtcy9r1b+26l/Kpd4o3S25t6eO3oE5hs9OKNiPGD7Xmmi758zfDvo0lfHBGa", + "nNd17YuoO4xhMmh1DSZXHQiWdLHg0Vp/McACApJcMTO/sEHcy3wERIF6nnuZu+juNuG+XjCfGpPhmxtX", + "zx9HfMlvIECxBD0/P3XHppQIMrFx8vIMcTaGZJ5wQLmrvTeCmGvJvj05bdvDAEVFku6Oj8w4gdjRKRGW", + "Pq5UGXD3sHfoGt4yA0Eyhgf4yH3VwlYMboudaVmEnoCDnQWd80Wn1K3dhDK1lazOpNBeNv1u11fthQl4", + "JYvGTecv7escPiPalC8FDk6ES8ZoxZA4VPmF+txJ52lK1Nzu3X2LkikkV+5Vx+VPeuWGfmfanPohP7ij", + "rVJBX2tv5n+Nndp12cw1LP+mhY+7vTuTsO/ARdh+ECQ3U6nYV6CW6eM7VOtKpqfCgBKEIw1qBir0U6pG", + "iAcf6+b38dPNp6renbgWssqkjui6ctkHe8cA2ryQdH5nW4xcJ7qpOyEbzm4aSOvf2QoCwCJCdiWQUVHs", + "9Fk90XORPPLo2oGiXxCKimbsA6LXI/o85xwRQVEoNKOyM1D1a51vjN74EMPBH/jqmH/pvi8wnxFFUjCg", + "tOPP7FpdCCwSKp+u1OHaqkhjOb5+akD5eFX1za+QesUf70AHSz3YPdK9V1qh7dbKULxDtXZ35aGKGxkP", + "MNkIk98gxLyF0JxnCGfVDUlPOWoneU/RFbhN6lOu8CFWbJP9VMW1NgFadGjuMQdauv27VRp0dype4C0m", + "8FDLCYfwh/TnJ4S0R5FLgFy2uugr1n3ctgnQAvN/Uw5UgG7naVDBeC9DnOtSWRDQkBJV4sjKrGinuu7u", + "1mftPD3aa/i4DKkhuqYD6XA50euKXoUYfpeunX3nuGo1LnlIzuVnZNeFDrRRQFJf+7u4eOWuG9tB1zmo", + "+YLn2M3BVT7Ltdrmr6VWN005E/5+vgKTK+FvB4G7zRrjHm7aRnj3Yl3bLUzJwBfTgRkI0/YSqIMqcqXW", + "Tsg4YWL9yGbKKScosHgwrO38skNkaVsepw6bMfMK/VDXxohmpu/8gH+06y6awn8zxI67z+6f9YkUY84S", + "g9oLjNhVMGHTOUFHcyRVtdu+T+APYF3szHnGsK8o/ot3K/EfGv3/aPwvdP9/bgGJVAoS4+/g7FdNupJO", + "VUz5wF3bWVyHaRXp+uXZWTwghBtUnW/+w+mmM9ziB+v3lH1FiBRL2wsrCz1yCuFixc4tTJYt/z0tuVvB", + "FVtwDr161ox77eofUtgHXN59sS/2pyS2KvXt1CrK60Y/i1XsOgKFNRDufoxSk8e+GKhHWrETI5cKgpVL", + "uytbHpfltd37b3gEp3CLdkexg4fK8BbNjoqw1rU6Std8f42O7/B9d6fcAmUrPd9Di+Onb3HMCh0uvNiW", + "TY37Szy2ammUKeduGxqXP088ZXovQ2m4XjIrQ9SqqvcuAdbdnVPcdQ/lco/PRb9BEWwr/RNHwFL0cFiu", + "pSeEIwoz4DJzv3H1Y3EL54qH+9GDjv+rAFOpjfuNCL75dPO/AAAA//8xTL49s04AAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index 5b794efd..c801c255 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -240,7 +240,7 @@ components: Image: type: object - required: [id, name, status, progress, created_at] + required: [id, name, status, created_at] properties: id: type: string @@ -255,12 +255,6 @@ components: enum: [pending, pulling, unpacking, converting, ready, failed] description: Build status example: ready - progress: - type: integer - minimum: 0 - maximum: 100 - description: Build progress percentage - example: 100 queue_position: type: integer description: Position in build queue (null if not queued) @@ -462,38 +456,6 @@ paths: schema: $ref: "#/components/schemas/Error" - /images/{id}/progress: - get: - summary: Stream image build progress (SSE) - operationId: getImageProgress - security: - - bearerAuth: [] - parameters: - - name: id - in: path - required: true - schema: - type: string - responses: - 200: - description: Progress event stream - content: - text/event-stream: - schema: - type: string - 404: - description: Image not found - content: - application/json: - schema: - $ref: "#/components/schemas/Error" - 500: - description: Internal server error - content: - application/json: - schema: - $ref: "#/components/schemas/Error" - /images/{id}: get: summary: Get image details From 0ee76468ae2664ed0c31ad0df8d2923f7e497851 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 7 Nov 2025 15:24:47 -0500 Subject: [PATCH 13/37] Avoid rate limit in CI --- .github/workflows/test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 36e6b71d..10944ca1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,14 @@ jobs: - name: Install dependencies run: go mod download + # Avoids rate limits when running the tests + # Tests includes pulling, then converting to disk images + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Run tests run: make test From a12d4ba4883b32ccc46e4064e0f9820ad7dffee8 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 7 Nov 2025 15:34:06 -0500 Subject: [PATCH 14/37] Add API level image test --- cmd/api/api/images_test.go | 95 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 cmd/api/api/images_test.go diff --git a/cmd/api/api/images_test.go b/cmd/api/api/images_test.go new file mode 100644 index 00000000..80b10210 --- /dev/null +++ b/cmd/api/api/images_test.go @@ -0,0 +1,95 @@ +package api + +import ( + "testing" + "time" + + "github.com/onkernel/hypeman/lib/images" + "github.com/onkernel/hypeman/lib/oapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestListImages_Empty(t *testing.T) { + svc := newTestService(t) + + resp, err := svc.ListImages(ctx(), oapi.ListImagesRequestObject{}) + require.NoError(t, err) + + list, ok := resp.(oapi.ListImages200JSONResponse) + require.True(t, ok, "expected 200 response") + assert.Empty(t, list) +} + +func TestGetImage_NotFound(t *testing.T) { + svc := newTestService(t) + + resp, err := svc.GetImage(ctx(), oapi.GetImageRequestObject{ + Id: "non-existent", + }) + require.NoError(t, err) + + notFound, ok := resp.(oapi.GetImage404JSONResponse) + require.True(t, ok, "expected 404 response") + assert.Equal(t, "not_found", notFound.Code) + assert.Equal(t, "image not found", notFound.Message) +} + +func TestCreateImage_Async(t *testing.T) { + svc := newTestService(t) + ctx := ctx() + + t.Log("Creating image...") + createResp, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ + Body: &oapi.CreateImageRequest{ + Name: "docker.io/library/alpine:latest", + }, + }) + require.NoError(t, err) + + acceptedResp, ok := createResp.(oapi.CreateImage202JSONResponse) + require.True(t, ok, "expected 202 accepted response") + + img := oapi.Image(acceptedResp) + require.Equal(t, "docker.io/library/alpine:latest", img.Name) + require.Equal(t, "img-alpine-latest", img.Id) + t.Logf("Image created: id=%s, initial_status=%s", img.Id, img.Status) + + // Poll until ready + t.Log("Polling for completion...") + for i := 0; i < 300; i++ { + getResp, err := svc.GetImage(ctx, oapi.GetImageRequestObject{Id: img.Id}) + require.NoError(t, err) + + imgResp, ok := getResp.(oapi.GetImage200JSONResponse) + require.True(t, ok, "expected 200 response") + + currentImg := oapi.Image(imgResp) + + if i%10 == 0 || currentImg.Status != img.Status { + t.Logf("Poll #%d: status=%s, queue_position=%v, has_size=%v", + i+1, currentImg.Status, currentImg.QueuePosition, currentImg.SizeBytes != nil) + } + + if currentImg.Status == oapi.ImageStatus(images.StatusReady) { + t.Log("Build complete!") + require.NotNil(t, currentImg.SizeBytes) + require.Greater(t, *currentImg.SizeBytes, int64(0)) + require.Nil(t, currentImg.Error) + return + } + + if currentImg.Status == oapi.ImageStatus(images.StatusFailed) { + errMsg := "" + if currentImg.Error != nil { + errMsg = *currentImg.Error + } + t.Fatalf("Build failed: %s", errMsg) + } + + time.Sleep(100 * time.Millisecond) + } + + t.Fatal("Build did not complete within 30 seconds") +} + From 5c5e07d432d2dcdcf12c8b70e88168c8180ea9ef Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 7 Nov 2025 15:43:41 -0500 Subject: [PATCH 15/37] Check queuing works --- cmd/api/api/images_test.go | 57 +++++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/cmd/api/api/images_test.go b/cmd/api/api/images_test.go index 80b10210..ce138fc6 100644 --- a/cmd/api/api/images_test.go +++ b/cmd/api/api/images_test.go @@ -1,6 +1,7 @@ package api import ( + "fmt" "testing" "time" @@ -39,7 +40,21 @@ func TestCreateImage_Async(t *testing.T) { svc := newTestService(t) ctx := ctx() - t.Log("Creating image...") + // Create images before alpine to populate the queue + t.Log("Creating image queue...") + queueImages := []string{ + "docker.io/library/busybox:latest", + "docker.io/library/nginx:alpine", + } + for _, name := range queueImages { + _, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ + Body: &oapi.CreateImageRequest{Name: name}, + }) + require.NoError(t, err) + } + + // Create alpine (should be last in queue) + t.Log("Creating alpine image (should be queued)...") createResp, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ Body: &oapi.CreateImageRequest{ Name: "docker.io/library/alpine:latest", @@ -53,11 +68,15 @@ func TestCreateImage_Async(t *testing.T) { img := oapi.Image(acceptedResp) require.Equal(t, "docker.io/library/alpine:latest", img.Name) require.Equal(t, "img-alpine-latest", img.Id) - t.Logf("Image created: id=%s, initial_status=%s", img.Id, img.Status) + t.Logf("Image created: id=%s, initial_status=%s, queue_position=%v", + img.Id, img.Status, img.QueuePosition) // Poll until ready t.Log("Polling for completion...") - for i := 0; i < 300; i++ { + lastStatus := img.Status + lastQueuePos := getQueuePos(img.QueuePosition) + + for i := 0; i < 3000; i++ { getResp, err := svc.GetImage(ctx, oapi.GetImageRequestObject{Id: img.Id}) require.NoError(t, err) @@ -65,10 +84,19 @@ func TestCreateImage_Async(t *testing.T) { require.True(t, ok, "expected 200 response") currentImg := oapi.Image(imgResp) + currentQueuePos := getQueuePos(currentImg.QueuePosition) - if i%10 == 0 || currentImg.Status != img.Status { - t.Logf("Poll #%d: status=%s, queue_position=%v, has_size=%v", - i+1, currentImg.Status, currentImg.QueuePosition, currentImg.SizeBytes != nil) + // Log when status or queue position changes + if currentImg.Status != lastStatus || currentQueuePos != lastQueuePos { + t.Logf("Update: status=%s, queue_position=%v", currentImg.Status, formatQueuePos(currentImg.QueuePosition)) + + // Queue position should only decrease (never increase) + if lastQueuePos > 0 && currentQueuePos > lastQueuePos { + t.Errorf("Queue position increased: %d -> %d", lastQueuePos, currentQueuePos) + } + + lastStatus = currentImg.Status + lastQueuePos = currentQueuePos } if currentImg.Status == oapi.ImageStatus(images.StatusReady) { @@ -87,9 +115,24 @@ func TestCreateImage_Async(t *testing.T) { t.Fatalf("Build failed: %s", errMsg) } - time.Sleep(100 * time.Millisecond) + time.Sleep(10 * time.Millisecond) } t.Fatal("Build did not complete within 30 seconds") } +func getQueuePos(pos *int) int { + if pos == nil { + return 0 + } + return *pos +} + +func formatQueuePos(pos *int) string { + if pos == nil { + return "none" + } + return fmt.Sprintf("%d", *pos) +} + + From 8bf8a4f02496349d72409d03ca7ce7577f0f513b Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 7 Nov 2025 15:49:04 -0500 Subject: [PATCH 16/37] Test failure on invalid tag --- cmd/api/api/images_test.go | 49 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/cmd/api/api/images_test.go b/cmd/api/api/images_test.go index ce138fc6..145ab1ec 100644 --- a/cmd/api/api/images_test.go +++ b/cmd/api/api/images_test.go @@ -121,6 +121,55 @@ func TestCreateImage_Async(t *testing.T) { t.Fatal("Build did not complete within 30 seconds") } +func TestCreateImage_InvalidTag(t *testing.T) { + svc := newTestService(t) + ctx := ctx() + + t.Log("Creating image with invalid tag...") + createResp, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ + Body: &oapi.CreateImageRequest{ + Name: "docker.io/library/busybox:foobar", + }, + }) + require.NoError(t, err) + + acceptedResp, ok := createResp.(oapi.CreateImage202JSONResponse) + require.True(t, ok, "expected 202 accepted response") + + img := oapi.Image(acceptedResp) + require.Equal(t, "docker.io/library/busybox:foobar", img.Name) + t.Logf("Image created: id=%s", img.Id) + + // Poll until failed + t.Log("Polling for failure...") + for i := 0; i < 1000; i++ { + getResp, err := svc.GetImage(ctx, oapi.GetImageRequestObject{Id: img.Id}) + require.NoError(t, err) + + imgResp, ok := getResp.(oapi.GetImage200JSONResponse) + require.True(t, ok, "expected 200 response") + + currentImg := oapi.Image(imgResp) + t.Logf("Status: %s", currentImg.Status) + + if currentImg.Status == oapi.ImageStatus(images.StatusFailed) { + t.Log("Build failed as expected") + require.NotNil(t, currentImg.Error) + require.Contains(t, *currentImg.Error, "foobar") + t.Logf("Error message: %s", *currentImg.Error) + return + } + + if currentImg.Status == oapi.ImageStatus(images.StatusReady) { + t.Fatal("Build should have failed but succeeded") + } + + time.Sleep(10 * time.Millisecond) + } + + t.Fatal("Build did not fail within timeout") +} + func getQueuePos(pos *int) int { if pos == nil { return 0 From d39fdcacfc780d7456a62719f90f7f923dddcca8 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 7 Nov 2025 16:21:57 -0500 Subject: [PATCH 17/37] Fix image filesystem layout --- cmd/api/api/images.go | 10 +- cmd/api/api/images_test.go | 13 ++- lib/images/manager.go | 87 ++++++---------- lib/images/manager_test.go | 69 +++++-------- lib/images/oci.go | 5 +- lib/images/queue.go | 46 ++++----- lib/images/storage.go | 90 +++++++++------- lib/oapi/oapi.go | 204 ++++++++++++++++++------------------- openapi.yaml | 22 ++-- 9 files changed, 248 insertions(+), 298 deletions(-) diff --git a/cmd/api/api/images.go b/cmd/api/api/images.go index e1419976..e5a228d7 100644 --- a/cmd/api/api/images.go +++ b/cmd/api/api/images.go @@ -47,11 +47,10 @@ func (s *ApiService) CreateImage(ctx context.Context, request oapi.CreateImageRe return oapi.CreateImage202JSONResponse(*img), nil } -// GetImage gets image details func (s *ApiService) GetImage(ctx context.Context, request oapi.GetImageRequestObject) (oapi.GetImageResponseObject, error) { log := logger.FromContext(ctx) - img, err := s.ImageManager.GetImage(ctx, request.Id) + img, err := s.ImageManager.GetImage(ctx, request.Name) if err != nil { switch { case errors.Is(err, images.ErrNotFound): @@ -60,7 +59,7 @@ func (s *ApiService) GetImage(ctx context.Context, request oapi.GetImageRequestO Message: "image not found", }, nil default: - log.Error("failed to get image", "error", err, "id", request.Id) + log.Error("failed to get image", "error", err, "name", request.Name) return oapi.GetImage500JSONResponse{ Code: "internal_error", Message: "failed to get image", @@ -70,11 +69,10 @@ func (s *ApiService) GetImage(ctx context.Context, request oapi.GetImageRequestO return oapi.GetImage200JSONResponse(*img), nil } -// DeleteImage deletes an image func (s *ApiService) DeleteImage(ctx context.Context, request oapi.DeleteImageRequestObject) (oapi.DeleteImageResponseObject, error) { log := logger.FromContext(ctx) - err := s.ImageManager.DeleteImage(ctx, request.Id) + err := s.ImageManager.DeleteImage(ctx, request.Name) if err != nil { switch { case errors.Is(err, images.ErrNotFound): @@ -83,7 +81,7 @@ func (s *ApiService) DeleteImage(ctx context.Context, request oapi.DeleteImageRe Message: "image not found", }, nil default: - log.Error("failed to delete image", "error", err, "id", request.Id) + log.Error("failed to delete image", "error", err, "name", request.Name) return oapi.DeleteImage500JSONResponse{ Code: "internal_error", Message: "failed to delete image", diff --git a/cmd/api/api/images_test.go b/cmd/api/api/images_test.go index 145ab1ec..0976826b 100644 --- a/cmd/api/api/images_test.go +++ b/cmd/api/api/images_test.go @@ -26,7 +26,7 @@ func TestGetImage_NotFound(t *testing.T) { svc := newTestService(t) resp, err := svc.GetImage(ctx(), oapi.GetImageRequestObject{ - Id: "non-existent", + Name: "non-existent:latest", }) require.NoError(t, err) @@ -67,9 +67,8 @@ func TestCreateImage_Async(t *testing.T) { img := oapi.Image(acceptedResp) require.Equal(t, "docker.io/library/alpine:latest", img.Name) - require.Equal(t, "img-alpine-latest", img.Id) - t.Logf("Image created: id=%s, initial_status=%s, queue_position=%v", - img.Id, img.Status, img.QueuePosition) + t.Logf("Image created: name=%s, initial_status=%s, queue_position=%v", + img.Name, img.Status, img.QueuePosition) // Poll until ready t.Log("Polling for completion...") @@ -77,7 +76,7 @@ func TestCreateImage_Async(t *testing.T) { lastQueuePos := getQueuePos(img.QueuePosition) for i := 0; i < 3000; i++ { - getResp, err := svc.GetImage(ctx, oapi.GetImageRequestObject{Id: img.Id}) + getResp, err := svc.GetImage(ctx, oapi.GetImageRequestObject{Name: img.Name}) require.NoError(t, err) imgResp, ok := getResp.(oapi.GetImage200JSONResponse) @@ -138,12 +137,12 @@ func TestCreateImage_InvalidTag(t *testing.T) { img := oapi.Image(acceptedResp) require.Equal(t, "docker.io/library/busybox:foobar", img.Name) - t.Logf("Image created: id=%s", img.Id) + t.Logf("Image created: name=%s", img.Name) // Poll until failed t.Log("Polling for failure...") for i := 0; i < 1000; i++ { - getResp, err := svc.GetImage(ctx, oapi.GetImageRequestObject{Id: img.Id}) + getResp, err := svc.GetImage(ctx, oapi.GetImageRequestObject{Name: img.Name}) require.NoError(t, err) imgResp, ok := getResp.(oapi.GetImage200JSONResponse) diff --git a/lib/images/manager.go b/lib/images/manager.go index 4ba6c33c..2f5047e9 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -5,9 +5,7 @@ import ( "fmt" "os" "path/filepath" - "regexp" "sort" - "strings" "time" "github.com/onkernel/hypeman/lib/oapi" @@ -16,7 +14,6 @@ import ( const ( StatusPending = "pending" StatusPulling = "pulling" - StatusUnpacking = "unpacking" StatusConverting = "converting" StatusReady = "ready" StatusFailed = "failed" @@ -63,30 +60,23 @@ func (m *manager) ListImages(ctx context.Context) ([]oapi.Image, error) { } func (m *manager) CreateImage(ctx context.Context, req oapi.CreateImageRequest) (*oapi.Image, error) { - imageID := req.Id - if imageID == nil || *imageID == "" { - generated := generateImageID(req.Name) - imageID = &generated - } - - if imageExists(m.dataDir, *imageID) { + if imageExists(m.dataDir, req.Name) { return nil, ErrAlreadyExists } meta := &imageMetadata{ - ID: *imageID, Name: req.Name, Status: StatusPending, Request: &req, CreatedAt: time.Now(), } - if err := writeMetadata(m.dataDir, *imageID, meta); err != nil { + if err := writeMetadata(m.dataDir, req.Name, meta); err != nil { return nil, fmt.Errorf("write initial metadata: %w", err) } - queuePos := m.queue.Enqueue(*imageID, req, func() { - m.buildImage(context.Background(), *imageID, req) + queuePos := m.queue.Enqueue(req.Name, req, func() { + m.buildImage(context.Background(), req.Name, req) }) img := meta.toOAPI() @@ -96,44 +86,42 @@ func (m *manager) CreateImage(ctx context.Context, req oapi.CreateImageRequest) return img, nil } -func (m *manager) buildImage(ctx context.Context, imageID string, req oapi.CreateImageRequest) { - defer m.queue.MarkComplete(imageID) +func (m *manager) buildImage(ctx context.Context, imageName string, req oapi.CreateImageRequest) { + defer m.queue.MarkComplete(imageName) - buildDir := filepath.Join(imageDir(m.dataDir, imageID), ".build") + buildDir := filepath.Join(imageDir(m.dataDir, imageName), ".build") tempDir := filepath.Join(buildDir, "rootfs") if err := os.MkdirAll(buildDir, 0755); err != nil { - m.updateStatus(imageID, StatusFailed, fmt.Errorf("create build dir: %w", err)) + m.updateStatus(imageName, StatusFailed, fmt.Errorf("create build dir: %w", err)) return } defer func() { - meta, _ := readMetadata(m.dataDir, imageID) + meta, _ := readMetadata(m.dataDir, imageName) if meta != nil && meta.Status == StatusReady { os.RemoveAll(buildDir) } }() - m.updateStatus(imageID, StatusPulling, nil) + m.updateStatus(imageName, StatusPulling, nil) containerMeta, err := m.ociClient.pullAndExport(ctx, req.Name, tempDir) if err != nil { - m.updateStatus(imageID, StatusFailed, fmt.Errorf("pull and export: %w", err)) + m.updateStatus(imageName, StatusFailed, fmt.Errorf("pull and export: %w", err)) return } - m.updateStatus(imageID, StatusUnpacking, nil) - - m.updateStatus(imageID, StatusConverting, nil) - diskPath := imagePath(m.dataDir, imageID) + m.updateStatus(imageName, StatusConverting, nil) + diskPath := imagePath(m.dataDir, imageName) diskSize, err := convertToExt4(tempDir, diskPath) if err != nil { - m.updateStatus(imageID, StatusFailed, fmt.Errorf("convert to ext4: %w", err)) + m.updateStatus(imageName, StatusFailed, fmt.Errorf("convert to ext4: %w", err)) return } - meta, err := readMetadata(m.dataDir, imageID) + meta, err := readMetadata(m.dataDir, imageName) if err != nil { - m.updateStatus(imageID, StatusFailed, fmt.Errorf("read metadata: %w", err)) + m.updateStatus(imageName, StatusFailed, fmt.Errorf("read metadata: %w", err)) return } @@ -145,14 +133,14 @@ func (m *manager) buildImage(ctx context.Context, imageID string, req oapi.Creat meta.Env = containerMeta.Env meta.WorkingDir = containerMeta.WorkingDir - if err := writeMetadata(m.dataDir, imageID, meta); err != nil { - m.updateStatus(imageID, StatusFailed, fmt.Errorf("write final metadata: %w", err)) + if err := writeMetadata(m.dataDir, imageName, meta); err != nil { + m.updateStatus(imageName, StatusFailed, fmt.Errorf("write final metadata: %w", err)) return } } -func (m *manager) updateStatus(imageID, status string, err error) { - meta, readErr := readMetadata(m.dataDir, imageID) +func (m *manager) updateStatus(imageName, status string, err error) { + meta, readErr := readMetadata(m.dataDir, imageName) if readErr != nil { return } @@ -163,7 +151,7 @@ func (m *manager) updateStatus(imageID, status string, err error) { meta.Error = &errorMsg } - writeMetadata(m.dataDir, imageID, meta) + writeMetadata(m.dataDir, imageName, meta) } func (m *manager) RecoverInterruptedBuilds() { @@ -179,51 +167,34 @@ func (m *manager) RecoverInterruptedBuilds() { for _, meta := range metas { switch meta.Status { - case StatusPending, StatusPulling, StatusUnpacking, StatusConverting: + case StatusPending, StatusPulling, StatusConverting: if meta.Request != nil { metaCopy := meta - m.queue.Enqueue(metaCopy.ID, *metaCopy.Request, func() { - m.buildImage(context.Background(), metaCopy.ID, *metaCopy.Request) + m.queue.Enqueue(metaCopy.Name, *metaCopy.Request, func() { + m.buildImage(context.Background(), metaCopy.Name, *metaCopy.Request) }) } } } } -func (m *manager) GetImage(ctx context.Context, id string) (*oapi.Image, error) { - meta, err := readMetadata(m.dataDir, id) +func (m *manager) GetImage(ctx context.Context, name string) (*oapi.Image, error) { + meta, err := readMetadata(m.dataDir, name) if err != nil { return nil, err } img := meta.toOAPI() - // Inject live queue position if pending if meta.Status == StatusPending { - img.QueuePosition = m.queue.GetPosition(id) + img.QueuePosition = m.queue.GetPosition(name) } return img, nil } -func (m *manager) DeleteImage(ctx context.Context, id string) error { - return deleteImage(m.dataDir, id) -} - -// generateImageID creates a valid ID from an image name -// Example: docker.io/library/nginx:latest -> img-nginx-latest -func generateImageID(imageName string) string { - // Extract image name and tag - parts := strings.Split(imageName, "/") - nameTag := parts[len(parts)-1] - - // Replace special characters with dashes - reg := regexp.MustCompile(`[^a-zA-Z0-9]+`) - sanitized := reg.ReplaceAllString(nameTag, "-") - sanitized = strings.Trim(sanitized, "-") - - // Add prefix - return "img-" + sanitized +func (m *manager) DeleteImage(ctx context.Context, name string) error { + return deleteImage(m.dataDir, name) } diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go index a369f6c3..ff1ca3be 100644 --- a/lib/images/manager_test.go +++ b/lib/images/manager_test.go @@ -27,22 +27,21 @@ func TestCreateImage(t *testing.T) { require.NoError(t, err) require.NotNil(t, img) require.Equal(t, "docker.io/library/alpine:latest", img.Name) - require.Equal(t, "img-alpine-latest", img.Id) - waitForReady(t, mgr, ctx, img.Id) + waitForReady(t, mgr, ctx, img.Name) - img, err = mgr.GetImage(ctx, img.Id) + img, err = mgr.GetImage(ctx, img.Name) require.NoError(t, err) require.Equal(t, oapi.ImageStatus(StatusReady), img.Status) require.NotNil(t, img.SizeBytes) require.Greater(t, *img.SizeBytes, int64(0)) - diskPath := filepath.Join(dataDir, "images", img.Id, "rootfs.ext4") + diskPath := filepath.Join(dataDir, "images", imageNameToPath(img.Name), "rootfs.ext4") _, err = os.Stat(diskPath) require.NoError(t, err) } -func TestCreateImageWithCustomID(t *testing.T) { +func TestCreateImageDifferentTag(t *testing.T) { ociClient, err := NewOCIClient(t.TempDir()) require.NoError(t, err) @@ -50,19 +49,16 @@ func TestCreateImageWithCustomID(t *testing.T) { mgr := NewManager(dataDir, ociClient, 1) ctx := context.Background() - customID := "my-custom-alpine" req := oapi.CreateImageRequest{ - Name: "docker.io/library/alpine:latest", - Id: &customID, + Name: "docker.io/library/alpine:3.18", } img, err := mgr.CreateImage(ctx, req) require.NoError(t, err) require.NotNil(t, img) - require.Equal(t, "my-custom-alpine", img.Id) + require.Equal(t, "docker.io/library/alpine:3.18", img.Name) - // Wait for build to complete - waitForReady(t, mgr, ctx, img.Id) + waitForReady(t, mgr, ctx, img.Name) } func TestCreateImageDuplicate(t *testing.T) { @@ -81,10 +77,9 @@ func TestCreateImageDuplicate(t *testing.T) { img1, err := mgr.CreateImage(ctx, req) require.NoError(t, err) - // Wait for build to start (moves from pending to pulling) - waitForReady(t, mgr, ctx, img1.Id) + waitForReady(t, mgr, ctx, img1.Name) - // Try to create duplicate (should fail even if first still building) + // Try to create duplicate _, err = mgr.CreateImage(ctx, req) require.ErrorIs(t, err, ErrAlreadyExists) } @@ -110,8 +105,7 @@ func TestListImages(t *testing.T) { img1, err := mgr.CreateImage(ctx, req1) require.NoError(t, err) - // Wait for build - waitForReady(t, mgr, ctx, img1.Id) + waitForReady(t, mgr, ctx, img1.Name) // List should return one image images, err = mgr.ListImages(ctx) @@ -136,14 +130,11 @@ func TestGetImage(t *testing.T) { created, err := mgr.CreateImage(ctx, req) require.NoError(t, err) - // Wait for build - waitForReady(t, mgr, ctx, created.Id) + waitForReady(t, mgr, ctx, created.Name) - // Get the image - img, err := mgr.GetImage(ctx, created.Id) + img, err := mgr.GetImage(ctx, created.Name) require.NoError(t, err) require.NotNil(t, img) - require.Equal(t, created.Id, img.Id) require.Equal(t, created.Name, img.Name) require.Equal(t, oapi.ImageStatus(StatusReady), img.Status) require.NotNil(t, img.SizeBytes) @@ -158,8 +149,7 @@ func TestGetImageNotFound(t *testing.T) { ctx := context.Background() - // Try to get non-existent image - _, err = mgr.GetImage(ctx, "nonexistent") + _, err = mgr.GetImage(ctx, "nonexistent:latest") require.ErrorIs(t, err, ErrNotFound) } @@ -178,19 +168,15 @@ func TestDeleteImage(t *testing.T) { created, err := mgr.CreateImage(ctx, req) require.NoError(t, err) - // Wait for ready - waitForReady(t, mgr, ctx, created.Id) + waitForReady(t, mgr, ctx, created.Name) - // Delete the image - err = mgr.DeleteImage(ctx, created.Id) + err = mgr.DeleteImage(ctx, created.Name) require.NoError(t, err) - // Verify it's gone - _, err = mgr.GetImage(ctx, created.Id) + _, err = mgr.GetImage(ctx, created.Name) require.ErrorIs(t, err, ErrNotFound) - // Verify files were removed - imageDir := filepath.Join(dataDir, "images", created.Id) + imageDir := filepath.Join(dataDir, "images", imageNameToPath(created.Name)) _, err = os.Stat(imageDir) require.True(t, os.IsNotExist(err)) } @@ -204,35 +190,34 @@ func TestDeleteImageNotFound(t *testing.T) { ctx := context.Background() - // Try to delete non-existent image - err = mgr.DeleteImage(ctx, "nonexistent") + err = mgr.DeleteImage(ctx, "nonexistent:latest") require.ErrorIs(t, err, ErrNotFound) } -func TestGenerateImageID(t *testing.T) { +func TestImageNameToPath(t *testing.T) { tests := []struct { input string expected string }{ - {"docker.io/library/nginx:latest", "img-nginx-latest"}, - {"docker.io/library/alpine:3.18", "img-alpine-3-18"}, - {"gcr.io/my-project/my-app:v1.0.0", "img-my-app-v1-0-0"}, - {"nginx", "img-nginx"}, - {"ubuntu:22.04", "img-ubuntu-22-04"}, + {"docker.io/library/nginx:latest", "docker.io/library/nginx/latest"}, + {"docker.io/library/alpine:3.18", "docker.io/library/alpine/3.18"}, + {"gcr.io/my-project/my-app:v1.0.0", "gcr.io/my-project/my-app/v1.0.0"}, + {"nginx", "nginx/latest"}, + {"ubuntu:22.04", "ubuntu/22.04"}, } for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { - result := generateImageID(tt.input) + result := imageNameToPath(tt.input) require.Equal(t, tt.expected, result) }) } } // waitForReady waits for an image build to complete -func waitForReady(t *testing.T, mgr Manager, ctx context.Context, imageID string) { +func waitForReady(t *testing.T, mgr Manager, ctx context.Context, imageName string) { for i := 0; i < 600; i++ { - img, err := mgr.GetImage(ctx, imageID) + img, err := mgr.GetImage(ctx, imageName) if err != nil { time.Sleep(100 * time.Millisecond) continue diff --git a/lib/images/oci.go b/lib/images/oci.go index 7d49f8df..c0a80f21 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -36,11 +36,11 @@ func (c *OCIClient) Close() error { } func (c *OCIClient) pullAndExport(ctx context.Context, imageRef, exportDir string) (*containerMetadata, error) { - ociLayoutDir := filepath.Join(c.cacheDir, fmt.Sprintf("oci-layout-%d", os.Getpid())) + // Use persistent OCI layout for caching (parse imageRef into path) + ociLayoutDir := filepath.Join(c.cacheDir, imageNameToPath(imageRef)) if err := os.MkdirAll(ociLayoutDir, 0755); err != nil { return nil, fmt.Errorf("create oci layout dir: %w", err) } - defer os.RemoveAll(ociLayoutDir) if err := c.pullToOCILayout(ctx, imageRef, ociLayoutDir); err != nil { return nil, fmt.Errorf("pull to oci layout: %w", err) @@ -223,7 +223,6 @@ func (c *OCIClient) unpackLayers(ctx context.Context, ociLayoutDir, targetDir st return nil } -// containerMetadata holds extracted container metadata type containerMetadata struct { Entrypoint []string Cmd []string diff --git a/lib/images/queue.go b/lib/images/queue.go index eab2f6f5..aae65684 100644 --- a/lib/images/queue.go +++ b/lib/images/queue.go @@ -6,22 +6,20 @@ import ( "github.com/onkernel/hypeman/lib/oapi" ) -// QueuedBuild represents a build waiting in queue type QueuedBuild struct { - ImageID string - Request oapi.CreateImageRequest - StartFn func() // Callback to start the build + ImageName string + Request oapi.CreateImageRequest + StartFn func() } // BuildQueue manages concurrent image builds with a configurable limit type BuildQueue struct { maxConcurrent int - active map[string]bool // imageID -> is building + active map[string]bool pending []QueuedBuild mu sync.Mutex } -// NewBuildQueue creates a new build queue with max concurrent limit func NewBuildQueue(maxConcurrent int) *BuildQueue { if maxConcurrent < 1 { maxConcurrent = 1 @@ -33,60 +31,50 @@ func NewBuildQueue(maxConcurrent int) *BuildQueue { } } -// Enqueue adds a build to the queue and returns queue position -// Returns 0 if build starts immediately, >0 if queued -func (q *BuildQueue) Enqueue(imageID string, req oapi.CreateImageRequest, startFn func()) int { +func (q *BuildQueue) Enqueue(imageName string, req oapi.CreateImageRequest, startFn func()) int { q.mu.Lock() defer q.mu.Unlock() build := QueuedBuild{ - ImageID: imageID, - Request: req, - StartFn: startFn, + ImageName: imageName, + Request: req, + StartFn: startFn, } - // If under limit, start immediately if len(q.active) < q.maxConcurrent { - q.active[imageID] = true + q.active[imageName] = true go startFn() - return 0 // Building now, not queued + return 0 } - // Otherwise, add to queue q.pending = append(q.pending, build) - return len(q.pending) // Position in queue + return len(q.pending) } -// MarkComplete marks a build as complete and starts the next queued build -func (q *BuildQueue) MarkComplete(imageID string) { +func (q *BuildQueue) MarkComplete(imageName string) { q.mu.Lock() defer q.mu.Unlock() - delete(q.active, imageID) + delete(q.active, imageName) - // Try to start next build if len(q.pending) > 0 && len(q.active) < q.maxConcurrent { next := q.pending[0] q.pending = q.pending[1:] - q.active[next.ImageID] = true + q.active[next.ImageName] = true go next.StartFn() } } -// GetPosition returns the queue position for an image -// Returns nil if not in queue (either building or complete) -func (q *BuildQueue) GetPosition(imageID string) *int { +func (q *BuildQueue) GetPosition(imageName string) *int { q.mu.Lock() defer q.mu.Unlock() - // Check if actively building - if q.active[imageID] { + if q.active[imageName] { return nil } - // Check if in queue for i, build := range q.pending { - if build.ImageID == imageID { + if build.ImageName == imageName { pos := i + 1 return &pos } diff --git a/lib/images/storage.go b/lib/images/storage.go index 3694bdd8..488e9c9e 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -5,13 +5,13 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "github.com/onkernel/hypeman/lib/oapi" ) type imageMetadata struct { - ID string `json:"id"` Name string `json:"name"` Status string `json:"status"` Error *string `json:"error,omitempty"` @@ -26,7 +26,6 @@ type imageMetadata struct { func (m *imageMetadata) toOAPI() *oapi.Image { img := &oapi.Image{ - Id: m.ID, Name: m.Name, Status: oapi.ImageStatus(m.Status), Error: m.Error, @@ -54,20 +53,35 @@ func (m *imageMetadata) toOAPI() *oapi.Image { return img } -func imageDir(dataDir, imageID string) string { - return filepath.Join(dataDir, "images", imageID) +// imageNameToPath converts image name to nested directory structure +// docker.io/library/alpine:latest → docker.io/library/alpine/latest +func imageNameToPath(name string) string { + // Split on last colon to separate tag + lastColon := strings.LastIndex(name, ":") + if lastColon == -1 { + // No tag, use "latest" + return filepath.Join(name, "latest") + } + + imagePath := name[:lastColon] + tag := name[lastColon+1:] + return filepath.Join(imagePath, tag) } -func imagePath(dataDir, imageID string) string { - return filepath.Join(imageDir(dataDir, imageID), "rootfs.ext4") +func imageDir(dataDir, imageName string) string { + return filepath.Join(dataDir, "images", imageNameToPath(imageName)) } -func metadataPath(dataDir, imageID string) string { - return filepath.Join(imageDir(dataDir, imageID), "metadata.json") +func imagePath(dataDir, imageName string) string { + return filepath.Join(imageDir(dataDir, imageName), "rootfs.ext4") } -func writeMetadata(dataDir, imageID string, meta *imageMetadata) error { - dir := imageDir(dataDir, imageID) +func metadataPath(dataDir, imageName string) string { + return filepath.Join(imageDir(dataDir, imageName), "metadata.json") +} + +func writeMetadata(dataDir, imageName string, meta *imageMetadata) error { + dir := imageDir(dataDir, imageName) if err := os.MkdirAll(dir, 0755); err != nil { return fmt.Errorf("create image directory: %w", err) } @@ -77,12 +91,12 @@ func writeMetadata(dataDir, imageID string, meta *imageMetadata) error { return fmt.Errorf("marshal metadata: %w", err) } - tempPath := metadataPath(dataDir, imageID) + ".tmp" + tempPath := metadataPath(dataDir, imageName) + ".tmp" if err := os.WriteFile(tempPath, data, 0644); err != nil { return fmt.Errorf("write temp metadata: %w", err) } - finalPath := metadataPath(dataDir, imageID) + finalPath := metadataPath(dataDir, imageName) if err := os.Rename(tempPath, finalPath); err != nil { os.Remove(tempPath) return fmt.Errorf("rename metadata: %w", err) @@ -91,8 +105,8 @@ func writeMetadata(dataDir, imageID string, meta *imageMetadata) error { return nil } -func readMetadata(dataDir, imageID string) (*imageMetadata, error) { - path := metadataPath(dataDir, imageID) +func readMetadata(dataDir, imageName string) (*imageMetadata, error) { + path := metadataPath(dataDir, imageName) data, err := os.ReadFile(path) if err != nil { if os.IsNotExist(err) { @@ -107,7 +121,7 @@ func readMetadata(dataDir, imageID string) (*imageMetadata, error) { } if meta.Status == StatusReady { - diskPath := imagePath(dataDir, imageID) + diskPath := imagePath(dataDir, imageName) if _, err := os.Stat(diskPath); err != nil { if os.IsNotExist(err) { return nil, fmt.Errorf("disk image missing: %s", diskPath) @@ -121,37 +135,45 @@ func readMetadata(dataDir, imageID string) (*imageMetadata, error) { func listMetadata(dataDir string) ([]*imageMetadata, error) { imagesDir := filepath.Join(dataDir, "images") - entries, err := os.ReadDir(imagesDir) - if err != nil { - if os.IsNotExist(err) { - return []*imageMetadata{}, nil - } - return nil, fmt.Errorf("read images directory: %w", err) - } - var metas []*imageMetadata - for _, entry := range entries { - if !entry.IsDir() { - continue - } - meta, err := readMetadata(dataDir, entry.Name()) + err := filepath.Walk(imagesDir, func(path string, info os.FileInfo, err error) error { if err != nil { - continue + return nil // Skip errors + } + + if info.Name() == "metadata.json" { + // Read metadata file + data, err := os.ReadFile(path) + if err != nil { + return nil + } + + var meta imageMetadata + if err := json.Unmarshal(data, &meta); err != nil { + return nil + } + + metas = append(metas, &meta) } - metas = append(metas, meta) + + return nil + }) + + if err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("walk images directory: %w", err) } return metas, nil } -func imageExists(dataDir, imageID string) bool { - _, err := readMetadata(dataDir, imageID) +func imageExists(dataDir, imageName string) bool { + _, err := readMetadata(dataDir, imageName) return err == nil } -func deleteImage(dataDir, imageID string) error { - dir := imageDir(dataDir, imageID) +func deleteImage(dataDir, imageName string) error { + dir := imageDir(dataDir, imageName) if _, err := os.Stat(dir); err != nil { if os.IsNotExist(err) { return ErrNotFound diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index a941e891..75847b34 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -39,7 +39,6 @@ const ( Pending ImageStatus = "pending" Pulling ImageStatus = "pulling" Ready ImageStatus = "ready" - Unpacking ImageStatus = "unpacking" ) // Defines values for InstanceState. @@ -69,9 +68,6 @@ type AttachVolumeRequest struct { // CreateImageRequest defines model for CreateImageRequest. type CreateImageRequest struct { - // Id Optional custom identifier (auto-generated if not provided) - Id *string `json:"id,omitempty"` - // Name OCI image reference (e.g., docker.io/library/nginx:latest) Name string `json:"name"` } @@ -168,10 +164,7 @@ type Image struct { // Error Error message if status is failed Error *string `json:"error"` - // Id Unique identifier - Id string `json:"id"` - - // Name OCI image reference + // Name OCI image reference (also serves as unique identifier) Name string `json:"name"` // QueuePosition Position in build queue (null if not queued) @@ -414,10 +407,10 @@ type ClientInterface interface { CreateImage(ctx context.Context, body CreateImageJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) // DeleteImage request - DeleteImage(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + DeleteImage(ctx context.Context, name string, reqEditors ...RequestEditorFn) (*http.Response, error) // GetImage request - GetImage(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + GetImage(ctx context.Context, name string, reqEditors ...RequestEditorFn) (*http.Response, error) // ListInstances request ListInstances(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -513,8 +506,8 @@ func (c *Client) CreateImage(ctx context.Context, body CreateImageJSONRequestBod return c.Client.Do(req) } -func (c *Client) DeleteImage(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewDeleteImageRequest(c.Server, id) +func (c *Client) DeleteImage(ctx context.Context, name string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewDeleteImageRequest(c.Server, name) if err != nil { return nil, err } @@ -525,8 +518,8 @@ func (c *Client) DeleteImage(ctx context.Context, id string, reqEditors ...Reque return c.Client.Do(req) } -func (c *Client) GetImage(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewGetImageRequest(c.Server, id) +func (c *Client) GetImage(ctx context.Context, name string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetImageRequest(c.Server, name) if err != nil { return nil, err } @@ -824,12 +817,12 @@ func NewCreateImageRequestWithBody(server string, contentType string, body io.Re } // NewDeleteImageRequest generates requests for DeleteImage -func NewDeleteImageRequest(server string, id string) (*http.Request, error) { +func NewDeleteImageRequest(server string, name string) (*http.Request, error) { var err error var pathParam0 string - pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "name", runtime.ParamLocationPath, name) if err != nil { return nil, err } @@ -858,12 +851,12 @@ func NewDeleteImageRequest(server string, id string) (*http.Request, error) { } // NewGetImageRequest generates requests for GetImage -func NewGetImageRequest(server string, id string) (*http.Request, error) { +func NewGetImageRequest(server string, name string) (*http.Request, error) { var err error var pathParam0 string - pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "name", runtime.ParamLocationPath, name) if err != nil { return nil, err } @@ -1451,10 +1444,10 @@ type ClientWithResponsesInterface interface { CreateImageWithResponse(ctx context.Context, body CreateImageJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateImageResponse, error) // DeleteImageWithResponse request - DeleteImageWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*DeleteImageResponse, error) + DeleteImageWithResponse(ctx context.Context, name string, reqEditors ...RequestEditorFn) (*DeleteImageResponse, error) // GetImageWithResponse request - GetImageWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetImageResponse, error) + GetImageWithResponse(ctx context.Context, name string, reqEditors ...RequestEditorFn) (*GetImageResponse, error) // ListInstancesWithResponse request ListInstancesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*ListInstancesResponse, error) @@ -1971,8 +1964,8 @@ func (c *ClientWithResponses) CreateImageWithResponse(ctx context.Context, body } // DeleteImageWithResponse request returning *DeleteImageResponse -func (c *ClientWithResponses) DeleteImageWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*DeleteImageResponse, error) { - rsp, err := c.DeleteImage(ctx, id, reqEditors...) +func (c *ClientWithResponses) DeleteImageWithResponse(ctx context.Context, name string, reqEditors ...RequestEditorFn) (*DeleteImageResponse, error) { + rsp, err := c.DeleteImage(ctx, name, reqEditors...) if err != nil { return nil, err } @@ -1980,8 +1973,8 @@ func (c *ClientWithResponses) DeleteImageWithResponse(ctx context.Context, id st } // GetImageWithResponse request returning *GetImageResponse -func (c *ClientWithResponses) GetImageWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetImageResponse, error) { - rsp, err := c.GetImage(ctx, id, reqEditors...) +func (c *ClientWithResponses) GetImageWithResponse(ctx context.Context, name string, reqEditors ...RequestEditorFn) (*GetImageResponse, error) { + rsp, err := c.GetImage(ctx, name, reqEditors...) if err != nil { return nil, err } @@ -2868,11 +2861,11 @@ type ServerInterface interface { // (POST /images) CreateImage(w http.ResponseWriter, r *http.Request) // Delete image - // (DELETE /images/{id}) - DeleteImage(w http.ResponseWriter, r *http.Request, id string) + // (DELETE /images/{name}) + DeleteImage(w http.ResponseWriter, r *http.Request, name string) // Get image details - // (GET /images/{id}) - GetImage(w http.ResponseWriter, r *http.Request, id string) + // (GET /images/{name}) + GetImage(w http.ResponseWriter, r *http.Request, name string) // List instances // (GET /instances) ListInstances(w http.ResponseWriter, r *http.Request) @@ -2937,14 +2930,14 @@ func (_ Unimplemented) CreateImage(w http.ResponseWriter, r *http.Request) { } // Delete image -// (DELETE /images/{id}) -func (_ Unimplemented) DeleteImage(w http.ResponseWriter, r *http.Request, id string) { +// (DELETE /images/{name}) +func (_ Unimplemented) DeleteImage(w http.ResponseWriter, r *http.Request, name string) { w.WriteHeader(http.StatusNotImplemented) } // Get image details -// (GET /images/{id}) -func (_ Unimplemented) GetImage(w http.ResponseWriter, r *http.Request, id string) { +// (GET /images/{name}) +func (_ Unimplemented) GetImage(w http.ResponseWriter, r *http.Request, name string) { w.WriteHeader(http.StatusNotImplemented) } @@ -3094,12 +3087,12 @@ func (siw *ServerInterfaceWrapper) DeleteImage(w http.ResponseWriter, r *http.Re var err error - // ------------- Path parameter "id" ------------- - var id string + // ------------- Path parameter "name" ------------- + var name string - err = runtime.BindStyledParameterWithOptions("simple", "id", chi.URLParam(r, "id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + err = runtime.BindStyledParameterWithOptions("simple", "name", chi.URLParam(r, "name"), &name, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) if err != nil { - siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "name", Err: err}) return } @@ -3110,7 +3103,7 @@ func (siw *ServerInterfaceWrapper) DeleteImage(w http.ResponseWriter, r *http.Re r = r.WithContext(ctx) handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - siw.Handler.DeleteImage(w, r, id) + siw.Handler.DeleteImage(w, r, name) })) for _, middleware := range siw.HandlerMiddlewares { @@ -3125,12 +3118,12 @@ func (siw *ServerInterfaceWrapper) GetImage(w http.ResponseWriter, r *http.Reque var err error - // ------------- Path parameter "id" ------------- - var id string + // ------------- Path parameter "name" ------------- + var name string - err = runtime.BindStyledParameterWithOptions("simple", "id", chi.URLParam(r, "id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + err = runtime.BindStyledParameterWithOptions("simple", "name", chi.URLParam(r, "name"), &name, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) if err != nil { - siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "name", Err: err}) return } @@ -3141,7 +3134,7 @@ func (siw *ServerInterfaceWrapper) GetImage(w http.ResponseWriter, r *http.Reque r = r.WithContext(ctx) handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - siw.Handler.GetImage(w, r, id) + siw.Handler.GetImage(w, r, name) })) for _, middleware := range siw.HandlerMiddlewares { @@ -3670,10 +3663,10 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Post(options.BaseURL+"/images", wrapper.CreateImage) }) r.Group(func(r chi.Router) { - r.Delete(options.BaseURL+"/images/{id}", wrapper.DeleteImage) + r.Delete(options.BaseURL+"/images/{name}", wrapper.DeleteImage) }) r.Group(func(r chi.Router) { - r.Get(options.BaseURL+"/images/{id}", wrapper.GetImage) + r.Get(options.BaseURL+"/images/{name}", wrapper.GetImage) }) r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/instances", wrapper.ListInstances) @@ -3813,7 +3806,7 @@ func (response CreateImage500JSONResponse) VisitCreateImageResponse(w http.Respo } type DeleteImageRequestObject struct { - Id string `json:"id"` + Name string `json:"name"` } type DeleteImageResponseObject interface { @@ -3847,7 +3840,7 @@ func (response DeleteImage500JSONResponse) VisitDeleteImageResponse(w http.Respo } type GetImageRequestObject struct { - Id string `json:"id"` + Name string `json:"name"` } type GetImageResponseObject interface { @@ -4422,10 +4415,10 @@ type StrictServerInterface interface { // (POST /images) CreateImage(ctx context.Context, request CreateImageRequestObject) (CreateImageResponseObject, error) // Delete image - // (DELETE /images/{id}) + // (DELETE /images/{name}) DeleteImage(ctx context.Context, request DeleteImageRequestObject) (DeleteImageResponseObject, error) // Get image details - // (GET /images/{id}) + // (GET /images/{name}) GetImage(ctx context.Context, request GetImageRequestObject) (GetImageResponseObject, error) // List instances // (GET /instances) @@ -4577,10 +4570,10 @@ func (sh *strictHandler) CreateImage(w http.ResponseWriter, r *http.Request) { } // DeleteImage operation middleware -func (sh *strictHandler) DeleteImage(w http.ResponseWriter, r *http.Request, id string) { +func (sh *strictHandler) DeleteImage(w http.ResponseWriter, r *http.Request, name string) { var request DeleteImageRequestObject - request.Id = id + request.Name = name handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { return sh.ssi.DeleteImage(ctx, request.(DeleteImageRequestObject)) @@ -4603,10 +4596,10 @@ func (sh *strictHandler) DeleteImage(w http.ResponseWriter, r *http.Request, id } // GetImage operation middleware -func (sh *strictHandler) GetImage(w http.ResponseWriter, r *http.Request, id string) { +func (sh *strictHandler) GetImage(w http.ResponseWriter, r *http.Request, name string) { var request GetImageRequestObject - request.Id = id + request.Name = name handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { return sh.ssi.GetImage(ctx, request.(GetImageRequestObject)) @@ -4985,59 +4978,60 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xce28TuRb/Kpbv/aNISfNoYSH3LyjsUmkLFYWudFkUOeOTxFuPPbU9gYD63a/8mMlM", - "xnkU2izZWwmJTMY+xz7ndx4+x+k3nMg0kwKE0XjwDetkCilxH58bQ5LppeR5Cu/gOgdt7NeZkhkow8AN", - "SmUuzDAjZmqfKOhEscwwKfAAnxMzRZ+noADNHBWkpzLnFI0AuXlAcQvDF5JmHPAAd1JhOpQYglvYzDP7", - "lTaKiQm+aWEFhErB557NmOTc4MGYcA2tJbZnljQiGtkpbTenpDeSkgMR+MZRvM6ZAooHH6vb+FQOlqO/", - "IDGW+YkCYuA0JZPVkmC0KYG37gPhKMm1kSliFIRhYwYKHZDcyPYEBChigCI2RkIalCk5YxToo5pkWDpp", - "iwkTX9qzXkw4gqQQ4X5yiphdM1IwBgUiAXQAh5PDFqIyuQJ1yGSHs5Eiat5x5AecGNCmznz92OZylkTr", - "1rZGqEIbIpLVcgUxs/8RSpkX5nntdUMWdRm8EjOmpEhBGDQjipERB13d3jf85u3LV8NXby7xwHKmeeKm", - "tvD523fv8QAfdbtdS7ex/pjCPwh2nUNVz2OpkJkCYmGf6KBQMRrNUUI4B7WkbKFNm4ySXv8opmun0SZn", - "B84K4wZ+kqmSKawAUAqpVPNhSr4M01HNxI67z540LIx8YWmeIj8LfWZmiqbSZDyfICbQ2Ysqc08gcGTC", - "wARUlWWdXa/bP15m94JoKHg1yPe7x09j5OMm8TpPiWhbx2CBgNygqqDSefuzVFdcEtqOCiqTygxTkmVM", - "THTE5UllUPEajZVM0VRqg4xEk9xbCzOQupn/VjDGA/yvzsIDd4L77Vg6Z55MBXtEKTJ3zywFmZuhhkQK", - "qmsSPHrS7S5L8L0f78CoE8KhbWT7KyiJNKREGJbUbOKXviXRlOksyfI6s/4ypzd5OgKF5BjNmDI54ejk", - "/EONeD9K2cWHiEB9+NFWgMTFo20l6Cf6GGatvynGJT/FbDAKgPA2ttppbQiK9xkKZpK3bYxs3yIQ+OVG", - "0e5IeenH6Gn2FYaTUZPkBftqfRqasAkZzU3dp/Yi6IlFhQX9mKhfKSVVU7iJpJEtPs8yzhJin9o6g4SN", - "WYLAUkB2AjpISTJlAkrbr0t1ROhQBXW2YjHFEMYj8HxeRqXALIxEB9bU0pwblnHw7/SjbbHrdv7SUYpZ", - "PxMC1BAK8dyCUgpaR6PHkl8s9lIOcZ6DwiifTKxIqqI7Y1ozMUGFdtGYAacDn3lszA6cNhcLW4mDsIct", - "0fC7/AyqzWEGvAoCb1F2salUgEqceKUtBeEZ4YwOmchyE4+YK0T5a67MFAokIDKyjtdmAF5hVSY+Zltb", - "H8tc0KiwGuJ4DYT7bLsuCW2IyUPGlKdWtvLKynPBTl5tVEcgElPDaZF3LCkgjTi7k7OXPvglUhjCBCiU", - "giEhty9X9BG7TBK3cNtiihJIpUByPP6PXUFpKk0vl3NucYoHRuXQNJDEOWk6JCayNPvOItrGUG1ImqGD", - "d7+eHB0dPau7hH63/7jd7bV7j9/3uoOu/fdf3MJjqVJLF1NioG2JxNABwqh5JpmIrOBV+W47GXV8At5e", - "0DzU0x8T0D3k1Nvs5Rs+f/7+tT3p5Vp1uEwI7+gRE4PKc/m4eOE++McRE9FcvHSGSyt1th9M1cZVj2/E", - "NBoTxpfOn1nOefh+YHciICmRIp0XWCHXSnK+zZngfo52P3Rma+HrHHIYZlIzz6KZ2fo3NuiPcsYpcjPQ", - "gZVJkbK4r+oJS3+l1Crpn0sDfBrRYPyS6SukQ7rhxgSeuTCMu0P+vMbx8dGTp790n/X6FWNlwjw5xlst", - "pXSjS4cQt+fwtlX62AwE9RHRosd/ykVGkiv/OZFiZg3LPbi1Wh/isVdzzsW7hmJmoHRUIz56GDJBNjaz", - "iVfsgmKp642otUceJiZDyiI29Id/iShTkBh7BtvC0nGHZNlm1mtS8FLSFWceDUvhXB2JTHcfBY5uGwXu", - "o3TREMH4mopYHsL5HF3nhFunQxGVKWGieQiolBsOnQveBjFToodakExPZUS6f0zBpUAEFWMQfGHa6FAN", - "Ybosh1SXEqp5y6W67/KqP0cNpQY5KcZsktsMOa3XT763ZLKC+mhPyyV3VBvJFJsRA0OWRfj5d+j0HBFK", - "FejasRX3nvUPe0+eHva63cNedxs70IaoVT7mwr77DgfzeKWD2WY5BjbJr/CYF26wmyWzbOUmZHarPfQ3", - "OMmNe4jWtmLFrCRAnoR67W3KV/dZsvI1J6CoGLG7ilWBgK2j5kUBmCVHWBSrHbnBn6KNfOGLDtDl2RkK", - "1NEoNy7vC2aADk64zCl6Pc9AzZiWCgli2AweWQrvciGYmFgKNgEniX3D50j579dPPie59tzt3Mw9rZ9x", - "Mc0NlZ+Fm6OnuUH2yS3ZbiEEpPUkvGEM0Bvp5oSVtpCQy5HNDyeCjubN4ctR8CAhAo1s7q6NVEAf/Skq", - "KWWQNG7hIDHcwn77uIWLXdmPfnXuk2Nc0fTCnKrespEiuZr00DrpFb6ZCVe6cOPQ5VnVKJ5GbWwq1xOU", - "nqAdVicWJ5cpaWQiea3ojE2SVeTln3KaRfa/ZDGL1bWqe49ZiLfGpshIsO6hkWvs5vSlPRIVY9ekJhvd", - "4V1nsd1nt61l3D77Wl+jXtcy9r1b+26l/Kpd4o3S25t6eO3oE5hs9OKNiPGD7Xmmi758zfDvo0lfHBGa", - "nNd17YuoO4xhMmh1DSZXHQiWdLHg0Vp/McACApJcMTO/sEHcy3wERIF6nnuZu+juNuG+XjCfGpPhmxtX", - "zx9HfMlvIECxBD0/P3XHppQIMrFx8vIMcTaGZJ5wQLmrvTeCmGvJvj05bdvDAEVFku6Oj8w4gdjRKRGW", - "Pq5UGXD3sHfoGt4yA0Eyhgf4yH3VwlYMboudaVmEnoCDnQWd80Wn1K3dhDK1lazOpNBeNv1u11fthQl4", - "JYvGTecv7escPiPalC8FDk6ES8ZoxZA4VPmF+txJ52lK1Nzu3X2LkikkV+5Vx+VPeuWGfmfanPohP7ij", - "rVJBX2tv5n+Nndp12cw1LP+mhY+7vTuTsO/ARdh+ECQ3U6nYV6CW6eM7VOtKpqfCgBKEIw1qBir0U6pG", - "iAcf6+b38dPNp6renbgWssqkjui6ctkHe8cA2ryQdH5nW4xcJ7qpOyEbzm4aSOvf2QoCwCJCdiWQUVHs", - "9Fk90XORPPLo2oGiXxCKimbsA6LXI/o85xwRQVEoNKOyM1D1a51vjN74EMPBH/jqmH/pvi8wnxFFUjCg", - "tOPP7FpdCCwSKp+u1OHaqkhjOb5+akD5eFX1za+QesUf70AHSz3YPdK9V1qh7dbKULxDtXZ35aGKGxkP", - "MNkIk98gxLyF0JxnCGfVDUlPOWoneU/RFbhN6lOu8CFWbJP9VMW1NgFadGjuMQdauv27VRp0dype4C0m", - "8FDLCYfwh/TnJ4S0R5FLgFy2uugr1n3ctgnQAvN/Uw5UgG7naVDBeC9DnOtSWRDQkBJV4sjKrGinuu7u", - "1mftPD3aa/i4DKkhuqYD6XA50euKXoUYfpeunX3nuGo1LnlIzuVnZNeFDrRRQFJf+7u4eOWuG9tB1zmo", - "+YLn2M3BVT7Ltdrmr6VWN005E/5+vgKTK+FvB4G7zRrjHm7aRnj3Yl3bLUzJwBfTgRkI0/YSqIMqcqXW", - "Tsg4YWL9yGbKKScosHgwrO38skNkaVsepw6bMfMK/VDXxohmpu/8gH+06y6awn8zxI67z+6f9YkUY84S", - "g9oLjNhVMGHTOUFHcyRVtdu+T+APYF3szHnGsK8o/ot3K/EfGv3/aPwvdP9/bgGJVAoS4+/g7FdNupJO", - "VUz5wF3bWVyHaRXp+uXZWTwghBtUnW/+w+mmM9ziB+v3lH1FiBRL2wsrCz1yCuFixc4tTJYt/z0tuVvB", - "FVtwDr161ox77eofUtgHXN59sS/2pyS2KvXt1CrK60Y/i1XsOgKFNRDufoxSk8e+GKhHWrETI5cKgpVL", - "uytbHpfltd37b3gEp3CLdkexg4fK8BbNjoqw1rU6Std8f42O7/B9d6fcAmUrPd9Di+Onb3HMCh0uvNiW", - "TY37Szy2ammUKeduGxqXP088ZXovQ2m4XjIrQ9SqqvcuAdbdnVPcdQ/lco/PRb9BEWwr/RNHwFL0cFiu", - "pSeEIwoz4DJzv3H1Y3EL54qH+9GDjv+rAFOpjfuNCL75dPO/AAAA//8xTL49s04AAA==", + "H4sIAAAAAAAC/+xc+28Tu/L/Vyx/v0cqUtI8WjiQ+1MpcKhEoaLQI10Oipz1JPHBa29tbyBU/d+v/NjN", + "btZ5FNpcem4lJLJZe8Yz/szDM06vcCLTTAoQRuPBFdbJFFLiPh4ZQ5LpheR5Cu/hMgdt7NeZkhkow8AN", + "SmUuzDAjZmqfKOhEscwwKfAAnxEzRV+noADNHBWkpzLnFI0AuXlAcQvDN5JmHPAAd1JhOpQYglvYzDP7", + "lTaKiQm+bmEFhErB557NmOTc4MGYcA2tJbanljQiGtkpbTenpDeSkgMR+NpRvMyZAooHn6pifC4Hy9Hf", + "kBjL/FgBMXCSkslqTQiSQlMH745PELPzkIIxKBAJoD3Yn+y3EJXJF1D7THY4Gymi5h0xYeLbgBMD2jyq", + "qWb92Ka+lsRza1sjmNCGiGS1bCBm9j9CKbNyEX5We93YrLoOXooZU1KkIAyaEcXIiIOuineF37578XL4", + "8u0FHljONE/c1BY+e/f+Ax7gg263a+k21s9oU+UfBbvMATEKwrAxA4XGUiEzBcSCnGgvU3LGKFA0mqOE", + "cA6qrm87sk1GSa9/EAOj29EmZweQCuM6yXTSTqZKptCe9WJEU0ilmg9T8m2YjmowP+w+e9JAOfnG0jxF", + "fhb6yswUTaXJeD5BTKDT51XmnkDgyISBCagqyzq7Xrd/uMzuOdFQ8GqQ73cPn8bIx03idZ4S0bbGaYGA", + "3KCqotJ5+6tUX7gktB1VVCaVGaYky5iY6Ijbkcqg4jUaK5miqdQGGYkmubcWZiB1M/9fwRgP8P91Fl6w", + "E1xgx9I59WQq2CNKkbl7ZinI3Aw1JFJQXdPgwZNud1mDH/x4B0adEA5tI9vfQUmkISXCsKRmE7/3LYmm", + "TmdJlteZ9Zc5vc3TESgkx2jGlMkJR8dnH2vE+1HKzkdHFOpDgLYKJC4mbKtBP9HHEWv9TTUu+SlmA0IA", + "hLex1U5rQ2CKeYZ3mfdeKMm1kWnVReyR3Mj2BAQoYoAiNkZCGlT4ibp3mEnetnEqDs846v1yo2h3pLz2", + "Y/Q0+w7DyahJ8px9tz4NTdiEjOam7lN7EfTEosKCfkzVL5WSqqncRNKIiEdZxllC7FNbZ5CwMUsQWArI", + "TkB7KUmmTEBp+3WtjggdqrCdrVhMMYTxCDyPyqgUmIWRaM+aWppzwzIO/p1+tC12neQvHKWY9TMhQA2h", + "UM8NKKWgdTR6LPnFQpZyiPMcFEb5ZGJVUlXdKdOaiQkqdheNGXA68JnHxuzA7eZiYStxEGTYEg1v5FdQ", + "bQ4z4FUQeIuyi02lAlTixG/aUhCeEc7okIksN/GIuUKVr3JlplAgAZGRdbw2A/AbVmXiY7a19bHMBY0q", + "q6GO10C4z3jrmtCGmDxkTHlqdSu/WH0u2MkvG7cjEIltw0mRdyxtQBpxdsenL3zwS6QwhAlQKAVDQn5d", + "rugTdpkkbuG2xRQlkEqB5Hj8L7uC0lSaXi7n3OIUD4zKoWkgiXPSdEhMZGn2nUW0jaHakDRDe+9fHR8c", + "HDyru4R+t/+43e21e48/9LqDrv33b9zCY6lSSxdTYqBticTQAcKoeSaZiKzgZfluOx11fALeXtDc19Of", + "U9Ad5NTbyHKFz44+vLanrVyrDpcJ4R09YmJQeS4fFy/cB/84YiKai5fOcGmlzvaDqdq46vGNmEZjwvjS", + "GTDLOQ/fD6wkApISKdJ5gRV63RR/owcxwrVNv9QMtD0w5ssnh587hLXwZQ45DDOpmV9FM1X1b2wUH+WM", + "U+RmoD0rZJGDuK/qGUh/pRoq+ZyL6z4vaDB+wfQXpEP+4MYEnrkwjLuT87zG8fHBk6e/d5/1+hXrY8I8", + "OcRbLaX0i0unCidzeNsqnWYGgvoQZ+HgPyVSzKx1uAe3PusIPIBqHrZ419iMGSgd3QUfAgyZIBtg2cRv", + "5oJiub8boWfPLUxMhpRFDOFP/xJRpiAx9iC1hbniDsmyzaxXZHaFYivOOBpWwrk4Ellu34sf3NSL30Xp", + "oaGC8SUVsTyC8zm6zAm37oAiKlPCRDOJr5QL9p0L3QYsU6KHWpBMT2VEu39OwaUwBBVjEHxj2uhQzWC6", + "LGdUlxIqYsvlri0rJb9iDaQGOSnGbJLbDDet1z9+tOSxgvronpY7bqm2kSk2IwaGLIvw8+/QyRkilCrQ", + "tWMn7j3r7/eePN3vdbv7ve42dqANUat8zLl99wMO5vFKB7PNcgxs0l/hMc/dYDdLZtlKIWR2Ixn6G5zk", + "RhmitalYMSoJkCeh3nqT8tNdlpx8zQgoKkbsruJUIGDrqHleAGbJERbFZkdu8JdoI1+4ogN0cXqKAnU0", + "yo1L84IZoL1jLnOKXs8zUDOmpUKCGDaDR5bC+1wIJiaWgk2gSWLf8DlS/vv1k89Irj13OzdzT+tnnE9z", + "Q+VX4eboaW6QfXJLtiKEgLSehDeMAXor3Zyw0hYScjmy+eFE0NG8OXw5Cu4lRKCRzea1kQroo79EJYMM", + "msYtHDSGW9iLj1u4kMp+9Ktznxzjyk4vzKnqLRspkqspD62TXuGbmXClBzcOXZxWjeJp1Mamcj1B6Qna", + "YXVicXKZkkYmkteKxtgkWUVf/imnWUT+JYtZrK5VlT1mId4amyojwbqHRq6xm5MX9gRUjF2Tmmx0h7ed", + "xXaf3bQWcfPsa32NeV3b1fc/7buV+qt2Wn/wPP0L1rOrvrxgstGLNyLGT7a4mS562zXDv4tGd3FEaHJe", + "1/kuou4whsmwq2swuepAsLQXCx6t9c11CwhIcsXM/NwGca/zERAF6ij3OnfR3Qnhvl4wnxqT4etrV48f", + "R3zJHyBAsQQdnZ24Y1NKBJnYOHlxijgbQzJPOKDc1c4bQcy1VN8dn7TtYYCiIkl3x0dmnELs6JQISx9X", + "Cgy4u9/bdw1rmYEgGcMDfOC+amGrBidiZ1oWkSfgYGdB53zRCXVrN6HMbDWrMym0102/2/VVd2ECXsmi", + "8dL5W/sSh8+INuVLgYNT4ZIxWjUkDlV+oT530nmaEjW3srtvUTKF5It71XH5k14p0BumzYkf8pMSbZUK", + "+lp5M/9rSGrXZTPXsPzrFj7s9m5Nw76DFmH7UZDcTKVi34Fapo9vcVtXMj0RBpQg3Bc9VeiHVI0QDz7V", + "ze/T5+vP1X136lroKpM6steVCzPYOwbQ5rmk81sTMXIl57ruhGw4u24grX9rKwgAiyjZlUBGRW3TZ/VE", + "z0XyyKNrBxv9nFBUNFMfEL0e0Wc554gIikKNGZW9gqpf61zZ1OLaBxkO/shXR/0L932B+owokoIBpd0K", + "lnT1/k0bRCKpzRN8CzIcQu1bFyuLzKtIaerIblUUtxyKPzdQf7iqUOdFoR4jhzvYrqV26z2Cid/dAhit", + "lVH7J/bfXw1c3Az8rf8qtJp+678iPGMCfjs4WlwQvBuwdHflIosrHQ/g2wi+PyAE3YXSnGsKh+UNWVc5", + "aieJV9GWuEnuVa7wIVhtk35V1bU2A1u0iO4wCVu6PrxVHnZ7W7zAW0zhoZgUqgAP+dcvCGmPIpeBuXR5", + "0dis+7jOFaPb5F8LzC+F4Ei4dAWK286sCtDtPLkqGN/LEOfaZBYENCRalTiyMtfa6V53d+uzdp4e3Wv4", + "uAypobqmA+lwOdHrqm6FGt5I10+/dVy1GrdMJOfyK7LrQnvaKCCpLz6en78ss/zLHNR8wXPs5uAqn+Vi", + "cfMnT6u7tpwJf8FfgcmV8DeTwF2HjXEPV3UjvHuxtvEWpmTgm+nADIRpew3UQRW5k2snZJwwsX5kM+WU", + "ExRYPBjWdn7ZIbK0LY9Th82YeYWGrOujRDPT937AP9p1F13p/zLEDrvP7p71sRRjzhKD2guM2FUwYdM5", + "QUdzJFW13X+fwB/AupDMecYgVxT/xbuV+A83Df7R+F/s/f+4BSRSKUiMvwR0v4rilXSqYsp77t7Q4j5O", + "q0jXL05P4wEhXOHqXPkPJ5vOcItfnd9R9hUhUiztXlhZaNJTCDc7dm5hsrxzcE8L+VZxhQjOoVfPmnGv", + "Xf1rCPcBl7df7Iv9PYitSn07tYryvtOvYhW7jkBhDYS7H8LU9HFfDNQjrZDEyKWCYOXW8MqWx0V5b/ju", + "Gx7BKdyg3VFI8FAZ3qLZUVHWulZH6ZrvrtHxA77v9ja3QNlKz/fQ4vjlWxyzYg8XXmzLpsbdJR5btTTK", + "lHO3DY2LXyeeMn0vQ2m4tDIrQ9SqqvcuAdbdnVPcdQ/l4h6fi/6AIthW+ieOgKUYu8X0RiaEIwoz4DJz", + "P7L1Y3EL54qHC9qDjv+zAlOpjfuRCr7+fP2fAAAA//9FztPZeE4AAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index c801c255..fd444d21 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -233,26 +233,18 @@ components: type: string description: OCI image reference (e.g., docker.io/library/nginx:latest) example: docker.io/library/nginx:latest - id: - type: string - description: Optional custom identifier (auto-generated if not provided) - example: img-nginx-v1 Image: type: object - required: [id, name, status, created_at] + required: [name, status, created_at] properties: - id: - type: string - description: Unique identifier - example: img-nginx-v1 name: type: string - description: OCI image reference + description: OCI image reference (also serves as unique identifier) example: docker.io/library/nginx:latest status: type: string - enum: [pending, pulling, unpacking, converting, ready, failed] + enum: [pending, pulling, converting, ready, failed] description: Build status example: ready queue_position: @@ -456,18 +448,19 @@ paths: schema: $ref: "#/components/schemas/Error" - /images/{id}: + /images/{name}: get: summary: Get image details operationId: getImage security: - bearerAuth: [] parameters: - - name: id + - name: name in: path required: true schema: type: string + description: URL-encoded image name (e.g. docker.io%2Flibrary%2Falpine%3Alatest) responses: 200: description: Image details @@ -493,11 +486,12 @@ paths: security: - bearerAuth: [] parameters: - - name: id + - name: name in: path required: true schema: type: string + description: URL-encoded image name responses: 204: description: Image deleted From 5fbacba52decf1b8b07fa1dee09e423a003eaf47 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 7 Nov 2025 17:32:29 -0500 Subject: [PATCH 18/37] image name validation --- cmd/api/api/images.go | 9 +++++-- cmd/api/api/images_test.go | 24 ++++++++++++++++++ lib/images/errors.go | 19 ++++++++++---- lib/images/manager.go | 45 ++++++++++++++++++++++++++------ lib/images/validation_test.go | 48 +++++++++++++++++++++++++++++++++++ 5 files changed, 130 insertions(+), 15 deletions(-) create mode 100644 lib/images/validation_test.go diff --git a/cmd/api/api/images.go b/cmd/api/api/images.go index e5a228d7..9cf198d3 100644 --- a/cmd/api/api/images.go +++ b/cmd/api/api/images.go @@ -31,6 +31,11 @@ func (s *ApiService) CreateImage(ctx context.Context, request oapi.CreateImageRe img, err := s.ImageManager.CreateImage(ctx, *request.Body) if err != nil { switch { + case images.IsInvalidNameError(err): + return oapi.CreateImage400JSONResponse{ + Code: "invalid_name", + Message: err.Error(), + }, nil case errors.Is(err, images.ErrAlreadyExists): return oapi.CreateImage400JSONResponse{ Code: "already_exists", @@ -53,7 +58,7 @@ func (s *ApiService) GetImage(ctx context.Context, request oapi.GetImageRequestO img, err := s.ImageManager.GetImage(ctx, request.Name) if err != nil { switch { - case errors.Is(err, images.ErrNotFound): + case images.IsInvalidNameError(err), errors.Is(err, images.ErrNotFound): return oapi.GetImage404JSONResponse{ Code: "not_found", Message: "image not found", @@ -75,7 +80,7 @@ func (s *ApiService) DeleteImage(ctx context.Context, request oapi.DeleteImageRe err := s.ImageManager.DeleteImage(ctx, request.Name) if err != nil { switch { - case errors.Is(err, images.ErrNotFound): + case images.IsInvalidNameError(err), errors.Is(err, images.ErrNotFound): return oapi.DeleteImage404JSONResponse{ Code: "not_found", Message: "image not found", diff --git a/cmd/api/api/images_test.go b/cmd/api/api/images_test.go index 0976826b..38527c4c 100644 --- a/cmd/api/api/images_test.go +++ b/cmd/api/api/images_test.go @@ -169,6 +169,30 @@ func TestCreateImage_InvalidTag(t *testing.T) { t.Fatal("Build did not fail within timeout") } +func TestCreateImage_InvalidName(t *testing.T) { + svc := newTestService(t) + ctx := ctx() + + invalidNames := []string{ + "invalid::", + "has spaces", + "", + } + + for _, name := range invalidNames { + t.Run(name, func(t *testing.T) { + createResp, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ + Body: &oapi.CreateImageRequest{Name: name}, + }) + require.NoError(t, err) + + badReq, ok := createResp.(oapi.CreateImage400JSONResponse) + require.True(t, ok, "expected 400 bad request for invalid name: %s", name) + require.Equal(t, "invalid_name", badReq.Code) + }) + } +} + func getQueuePos(pos *int) int { if pos == nil { return 0 diff --git a/lib/images/errors.go b/lib/images/errors.go index 030a9c4f..defd952d 100644 --- a/lib/images/errors.go +++ b/lib/images/errors.go @@ -1,11 +1,20 @@ package images -import "errors" +import ( + "errors" + "strings" +) var ( - ErrNotFound = errors.New("image not found") - ErrAlreadyExists = errors.New("image already exists") - ErrInvalidImage = errors.New("invalid image reference") - ErrConversionFailed = errors.New("failed to convert image") + ErrNotFound = errors.New("image not found") + ErrAlreadyExists = errors.New("image already exists") ) +// IsInvalidNameError checks if an error is due to invalid image name +func IsInvalidNameError(err error) bool { + if err == nil { + return false + } + return strings.Contains(err.Error(), "invalid image name") || + strings.Contains(err.Error(), "invalid reference format") +} diff --git a/lib/images/manager.go b/lib/images/manager.go index 2f5047e9..5b85f549 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -8,6 +8,7 @@ import ( "sort" "time" + "github.com/distribution/reference" "github.com/onkernel/hypeman/lib/oapi" ) @@ -60,23 +61,28 @@ func (m *manager) ListImages(ctx context.Context) ([]oapi.Image, error) { } func (m *manager) CreateImage(ctx context.Context, req oapi.CreateImageRequest) (*oapi.Image, error) { - if imageExists(m.dataDir, req.Name) { + normalizedName, err := normalizeImageName(req.Name) + if err != nil { + return nil, fmt.Errorf("invalid image name: %w", err) + } + + if imageExists(m.dataDir, normalizedName) { return nil, ErrAlreadyExists } meta := &imageMetadata{ - Name: req.Name, + Name: normalizedName, Status: StatusPending, Request: &req, CreatedAt: time.Now(), } - if err := writeMetadata(m.dataDir, req.Name, meta); err != nil { + if err := writeMetadata(m.dataDir, normalizedName, meta); err != nil { return nil, fmt.Errorf("write initial metadata: %w", err) } - queuePos := m.queue.Enqueue(req.Name, req, func() { - m.buildImage(context.Background(), req.Name, req) + queuePos := m.queue.Enqueue(normalizedName, req, func() { + m.buildImage(context.Background(), normalizedName, req) }) img := meta.toOAPI() @@ -179,7 +185,12 @@ func (m *manager) RecoverInterruptedBuilds() { } func (m *manager) GetImage(ctx context.Context, name string) (*oapi.Image, error) { - meta, err := readMetadata(m.dataDir, name) + normalizedName, err := normalizeImageName(name) + if err != nil { + return nil, fmt.Errorf("invalid image name: %w", err) + } + + meta, err := readMetadata(m.dataDir, normalizedName) if err != nil { return nil, err } @@ -187,14 +198,32 @@ func (m *manager) GetImage(ctx context.Context, name string) (*oapi.Image, error img := meta.toOAPI() if meta.Status == StatusPending { - img.QueuePosition = m.queue.GetPosition(name) + img.QueuePosition = m.queue.GetPosition(normalizedName) } return img, nil } func (m *manager) DeleteImage(ctx context.Context, name string) error { - return deleteImage(m.dataDir, name) + normalizedName, err := normalizeImageName(name) + if err != nil { + return fmt.Errorf("invalid image name: %w", err) + } + return deleteImage(m.dataDir, normalizedName) +} + +// normalizeImageName validates and normalizes an OCI image reference +// Examples: alpine → docker.io/library/alpine:latest +// nginx:1.0 → docker.io/library/nginx:1.0 +func normalizeImageName(name string) (string, error) { + named, err := reference.ParseNormalizedNamed(name) + if err != nil { + return "", err + } + + // Ensure it has a tag (add :latest if missing) + tagged := reference.TagNameOnly(named) + return tagged.String(), nil } diff --git a/lib/images/validation_test.go b/lib/images/validation_test.go new file mode 100644 index 00000000..9b37e074 --- /dev/null +++ b/lib/images/validation_test.go @@ -0,0 +1,48 @@ +package images + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNormalizeImageName(t *testing.T) { + tests := []struct { + input string + expected string + wantErr bool + }{ + // Valid images with full reference + {"docker.io/library/alpine:latest", "docker.io/library/alpine:latest", false}, + {"ghcr.io/myorg/myapp:v1.0.0", "ghcr.io/myorg/myapp:v1.0.0", false}, + + // Shorthand (gets expanded) + {"alpine", "docker.io/library/alpine:latest", false}, + {"alpine:3.18", "docker.io/library/alpine:3.18", false}, + {"nginx", "docker.io/library/nginx:latest", false}, + {"nginx:alpine", "docker.io/library/nginx:alpine", false}, + + // Without tag (gets :latest added) + {"docker.io/library/alpine", "docker.io/library/alpine:latest", false}, + {"ubuntu", "docker.io/library/ubuntu:latest", false}, + + // Invalid + {"", "", true}, + {"invalid::", "", true}, + {"has spaces", "", true}, + {"UPPERCASE", "", true}, // Repository names must be lowercase + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result, err := normalizeImageName(tt.input) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tt.expected, result) + } + }) + } +} + From bc8a7e516a719c71523b9350ee18d0302591af7c Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 7 Nov 2025 17:38:17 -0500 Subject: [PATCH 19/37] Update README --- lib/images/README.md | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/lib/images/README.md b/lib/images/README.md index 74933239..015b54c9 100644 --- a/lib/images/README.md +++ b/lib/images/README.md @@ -1,6 +1,6 @@ # Image Manager -Converts OCI container images into bootable ext4 disk images for Cloud Hypervisor VMs. +Converts OCI images to bootable ext4 disks for Cloud Hypervisor VMs. ## Architecture @@ -46,23 +46,35 @@ OCI Registry → containers/image → OCI Layout → umoci → rootfs/ → mkfs. - `-O ^has_journal` - No journal (disks mounted read-only in VMs, saves ~32MB) - Minimum 10MB size covers ext4 metadata (~5MB for superblock, inodes, bitmaps) -**Alternative tried:** go-diskfs pure Go ext4 - has bugs +**Alternative tried:** go-diskfs pure Go ext4, got too complicated. Could revisit this. -**Tradeoff:** Shell command vs pure Go, but mkfs.ext4 is universally available and robust +**Tradeoff:** Shell command vs pure Go, but mkfs.ext4 is widely available and robust + +## Filesystem Layout (storage.go) -## Filesystem Persistence (storage.go) -**Metadata:** JSON files with atomic writes (tmp + rename) ``` -/var/lib/hypeman/images/{id}/ - rootfs.ext4 - metadata.json +/var/lib/hypeman/ + images/ + docker.io/library/alpine/ + latest/ + metadata.json # Status, entrypoint, cmd, env + rootfs.ext4 # Bootable disk + 3.18/ # Different version + system/oci-cache/ + docker.io/library/alpine/latest/ + blobs/sha256/... # Shared layers, persistent ``` -**Why filesystem vs database?** -- Disk images must be on filesystem anyway -- No sync issues between DB and actual artifacts -- Simple recovery (scan directory to rebuild state) +**Benefits:** +- Natural hierarchy (versions grouped) +- Layer caching (alpine:latest and alpine:3.18 share base layers) + +## Input Validation + +Uses `github.com/distribution/reference` to validate and normalize names: +- `alpine` → `docker.io/library/alpine:latest` +- Rejects invalid formats (returns 400) ## Build Tags From 6cc60a60b448a66d30644133aa426bf3e6184836 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 09:39:50 -0500 Subject: [PATCH 20/37] Clean up error --- cmd/api/api/images.go | 6 +++--- lib/images/errors.go | 15 ++------------- lib/images/manager.go | 6 +++--- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/cmd/api/api/images.go b/cmd/api/api/images.go index 9cf198d3..0b8ecd32 100644 --- a/cmd/api/api/images.go +++ b/cmd/api/api/images.go @@ -31,7 +31,7 @@ func (s *ApiService) CreateImage(ctx context.Context, request oapi.CreateImageRe img, err := s.ImageManager.CreateImage(ctx, *request.Body) if err != nil { switch { - case images.IsInvalidNameError(err): + case errors.Is(err, images.ErrInvalidName): return oapi.CreateImage400JSONResponse{ Code: "invalid_name", Message: err.Error(), @@ -58,7 +58,7 @@ func (s *ApiService) GetImage(ctx context.Context, request oapi.GetImageRequestO img, err := s.ImageManager.GetImage(ctx, request.Name) if err != nil { switch { - case images.IsInvalidNameError(err), errors.Is(err, images.ErrNotFound): + case errors.Is(err, images.ErrInvalidName), errors.Is(err, images.ErrNotFound): return oapi.GetImage404JSONResponse{ Code: "not_found", Message: "image not found", @@ -80,7 +80,7 @@ func (s *ApiService) DeleteImage(ctx context.Context, request oapi.DeleteImageRe err := s.ImageManager.DeleteImage(ctx, request.Name) if err != nil { switch { - case images.IsInvalidNameError(err), errors.Is(err, images.ErrNotFound): + case errors.Is(err, images.ErrInvalidName), errors.Is(err, images.ErrNotFound): return oapi.DeleteImage404JSONResponse{ Code: "not_found", Message: "image not found", diff --git a/lib/images/errors.go b/lib/images/errors.go index defd952d..acabf3bd 100644 --- a/lib/images/errors.go +++ b/lib/images/errors.go @@ -1,20 +1,9 @@ package images -import ( - "errors" - "strings" -) +import "errors" var ( ErrNotFound = errors.New("image not found") ErrAlreadyExists = errors.New("image already exists") + ErrInvalidName = errors.New("invalid image name") ) - -// IsInvalidNameError checks if an error is due to invalid image name -func IsInvalidNameError(err error) bool { - if err == nil { - return false - } - return strings.Contains(err.Error(), "invalid image name") || - strings.Contains(err.Error(), "invalid reference format") -} diff --git a/lib/images/manager.go b/lib/images/manager.go index 5b85f549..5b0eaffa 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -63,7 +63,7 @@ func (m *manager) ListImages(ctx context.Context) ([]oapi.Image, error) { func (m *manager) CreateImage(ctx context.Context, req oapi.CreateImageRequest) (*oapi.Image, error) { normalizedName, err := normalizeImageName(req.Name) if err != nil { - return nil, fmt.Errorf("invalid image name: %w", err) + return nil, fmt.Errorf("%w: %s", ErrInvalidName, err.Error()) } if imageExists(m.dataDir, normalizedName) { @@ -187,7 +187,7 @@ func (m *manager) RecoverInterruptedBuilds() { func (m *manager) GetImage(ctx context.Context, name string) (*oapi.Image, error) { normalizedName, err := normalizeImageName(name) if err != nil { - return nil, fmt.Errorf("invalid image name: %w", err) + return nil, fmt.Errorf("%w: %s", ErrInvalidName, err.Error()) } meta, err := readMetadata(m.dataDir, normalizedName) @@ -207,7 +207,7 @@ func (m *manager) GetImage(ctx context.Context, name string) (*oapi.Image, error func (m *manager) DeleteImage(ctx context.Context, name string) error { normalizedName, err := normalizeImageName(name) if err != nil { - return fmt.Errorf("invalid image name: %w", err) + return fmt.Errorf("%w: %s", ErrInvalidName, err.Error()) } return deleteImage(m.dataDir, normalizedName) } From 675e09028c89736ad45a9954ce8b744ac25086d8 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 09:50:09 -0500 Subject: [PATCH 21/37] Use erofs --- README.md | 4 ++-- lib/images/README.md | 26 ++++++++++++------------- lib/images/disk.go | 40 +++++++------------------------------- lib/images/manager.go | 4 ++-- lib/images/manager_test.go | 2 +- lib/images/storage.go | 2 +- 6 files changed, 25 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 286fbc10..771d2d58 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ Run containerized workloads in VMs, powered by [Cloud Hypervisor](https://github ### Prerequisites -**Go 1.25.4+**, **Cloud Hypervisor**, **KVM**, **mkfs.ext4** +**Go 1.25.4+**, **Cloud Hypervisor**, **KVM**, **erofs-utils** ```bash cloud-hypervisor --version -mkfs.ext4 -V +mkfs.erofs --version ``` ### Configuration diff --git a/lib/images/README.md b/lib/images/README.md index 015b54c9..efbd468a 100644 --- a/lib/images/README.md +++ b/lib/images/README.md @@ -1,11 +1,11 @@ # Image Manager -Converts OCI images to bootable ext4 disks for Cloud Hypervisor VMs. +Converts OCI images to bootable erofs disks for Cloud Hypervisor VMs. ## Architecture ``` -OCI Registry → containers/image → OCI Layout → umoci → rootfs/ → mkfs.ext4 → disk.ext4 +OCI Registry → containers/image → OCI Layout → umoci → rootfs/ → mkfs.erofs → disk.erofs ``` ## Design Decisions @@ -32,23 +32,21 @@ OCI Registry → containers/image → OCI Layout → umoci → rootfs/ → mkfs. **Alternative:** With Docker API, the daemon (running as root) mounts image layers using overlayfs, then exports the merged filesystem. Users get the result without needing root themselves but it still has the dependency on Docker and does actually mount the overlays to get the merged filesystem. With umoci, layers are merged in userspace by extracting each tar layer sequentially and applying changes (including whiteouts for deletions). No kernel mount needed, fully rootless. Umoci was chosen because it's purpose-built for this use case and embeddable with the go program. -### Why mkfs.ext4? (disk.go) +### Why erofs? (disk.go) -**What:** Shell command to create ext4 filesystem +**What:** erofs (Enhanced Read-Only File System) with LZ4 compression **Why:** -- Battle-tested, fast, reliable -- Works without root when creating filesystem in a regular file (not block device) -- `-d` flag copies directory contents into filesystem +- Purpose-built for read-only overlay lowerdir +- Fast compression (~20-25% space savings) +- Fast decompression at VM boot +- Lower memory footprint than ext4 +- No journal/inode overhead **Options:** -- `-b 4096` - 4KB blocks (standard, matches VM page size) -- `-O ^has_journal` - No journal (disks mounted read-only in VMs, saves ~32MB) -- Minimum 10MB size covers ext4 metadata (~5MB for superblock, inodes, bitmaps) +- `-zlz4` - Fast compression (good balance for development) -**Alternative tried:** go-diskfs pure Go ext4, got too complicated. Could revisit this. - -**Tradeoff:** Shell command vs pure Go, but mkfs.ext4 is widely available and robust +**Alternative:** ext4 without journal works but erofs is optimized for this exact use case ## Filesystem Layout (storage.go) @@ -59,7 +57,7 @@ OCI Registry → containers/image → OCI Layout → umoci → rootfs/ → mkfs. docker.io/library/alpine/ latest/ metadata.json # Status, entrypoint, cmd, env - rootfs.ext4 # Bootable disk + rootfs.erofs # Compressed read-only disk 3.18/ # Different version system/oci-cache/ docker.io/library/alpine/latest/ diff --git a/lib/images/disk.go b/lib/images/disk.go index 09445b95..15ec66bf 100644 --- a/lib/images/disk.go +++ b/lib/images/disk.go @@ -7,46 +7,20 @@ import ( "path/filepath" ) -// convertToExt4 converts a rootfs directory to an ext4 disk image using mkfs.ext4 -func convertToExt4(rootfsDir, diskPath string) (int64, error) { - // Calculate size of rootfs directory - sizeBytes, err := dirSize(rootfsDir) - if err != nil { - return 0, fmt.Errorf("calculate dir size: %w", err) - } - - // Add 20% overhead for filesystem metadata, minimum 10MB - diskSizeBytes := sizeBytes + (sizeBytes / 5) - const minSize = 10 * 1024 * 1024 // 10MB - if diskSizeBytes < minSize { - diskSizeBytes = minSize - } - +// convertToErofs converts a rootfs directory to an erofs disk image using mkfs.erofs +func convertToErofs(rootfsDir, diskPath string) (int64, error) { // Ensure parent directory exists if err := os.MkdirAll(filepath.Dir(diskPath), 0755); err != nil { return 0, fmt.Errorf("create disk parent dir: %w", err) } - // Create sparse file - f, err := os.Create(diskPath) - if err != nil { - return 0, fmt.Errorf("create disk file: %w", err) - } - if err := f.Truncate(diskSizeBytes); err != nil { - f.Close() - return 0, fmt.Errorf("truncate disk file: %w", err) - } - f.Close() - - // Format as ext4 with rootfs contents using mkfs.ext4 - // -b 4096: 4KB blocks (standard, matches VM page size) - // -O ^has_journal: Disable journal (not needed for read-only VM mounts) - // -d: Copy directory contents into filesystem - // -F: Force creation (file not block device) - cmd := exec.Command("mkfs.ext4", "-b", "4096", "-O", "^has_journal", "-d", rootfsDir, "-F", diskPath) + // Create erofs image with LZ4 fast compression + // -zlz4: LZ4 fast compression (~20-25% space savings, faster builds) + // erofs doesn't need pre-allocation, creates file directly + cmd := exec.Command("mkfs.erofs", "-zlz4", diskPath, rootfsDir) output, err := cmd.CombinedOutput() if err != nil { - return 0, fmt.Errorf("mkfs.ext4 failed: %w, output: %s", err, output) + return 0, fmt.Errorf("mkfs.erofs failed: %w, output: %s", err, output) } // Get actual disk size diff --git a/lib/images/manager.go b/lib/images/manager.go index 5b0eaffa..69c63407 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -119,9 +119,9 @@ func (m *manager) buildImage(ctx context.Context, imageName string, req oapi.Cre m.updateStatus(imageName, StatusConverting, nil) diskPath := imagePath(m.dataDir, imageName) - diskSize, err := convertToExt4(tempDir, diskPath) + diskSize, err := convertToErofs(tempDir, diskPath) if err != nil { - m.updateStatus(imageName, StatusFailed, fmt.Errorf("convert to ext4: %w", err)) + m.updateStatus(imageName, StatusFailed, fmt.Errorf("convert to erofs: %w", err)) return } diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go index ff1ca3be..18d69d78 100644 --- a/lib/images/manager_test.go +++ b/lib/images/manager_test.go @@ -36,7 +36,7 @@ func TestCreateImage(t *testing.T) { require.NotNil(t, img.SizeBytes) require.Greater(t, *img.SizeBytes, int64(0)) - diskPath := filepath.Join(dataDir, "images", imageNameToPath(img.Name), "rootfs.ext4") + diskPath := filepath.Join(dataDir, "images", imageNameToPath(img.Name), "rootfs.erofs") _, err = os.Stat(diskPath) require.NoError(t, err) } diff --git a/lib/images/storage.go b/lib/images/storage.go index 488e9c9e..5b0b3cf8 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -73,7 +73,7 @@ func imageDir(dataDir, imageName string) string { } func imagePath(dataDir, imageName string) string { - return filepath.Join(imageDir(dataDir, imageName), "rootfs.ext4") + return filepath.Join(imageDir(dataDir, imageName), "rootfs.erofs") } func metadataPath(dataDir, imageName string) string { From e68a49d615e9d8ea8db9f2a3433e8d24e5feb14b Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 10:21:45 -0500 Subject: [PATCH 22/37] API layer depends on domain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cmd/api/api (handlers) ↓ depends on lib/images, lib/instances, lib/volumes (domain) ↓ independent of lib/oapi (API types) --- cmd/api/api/images.go | 47 ++++++++++++++++++++---- cmd/api/api/instances.go | 48 +++++++++++++++++++------ cmd/api/api/volumes.go | 31 +++++++++++++--- lib/images/manager.go | 26 +++++++------- lib/images/manager_test.go | 25 ++++++------- lib/images/queue.go | 10 ++---- lib/images/storage.go | 36 +++++++++---------- lib/images/types.go | 23 ++++++++++++ lib/instances/manager.go | 74 +++++++++----------------------------- lib/instances/types.go | 21 +++++++++++ lib/volumes/manager.go | 40 +++++---------------- lib/volumes/types.go | 17 +++++++++ 12 files changed, 233 insertions(+), 165 deletions(-) create mode 100644 lib/images/types.go create mode 100644 lib/instances/types.go create mode 100644 lib/volumes/types.go diff --git a/cmd/api/api/images.go b/cmd/api/api/images.go index 0b8ecd32..55210c3f 100644 --- a/cmd/api/api/images.go +++ b/cmd/api/api/images.go @@ -9,11 +9,10 @@ import ( "github.com/onkernel/hypeman/lib/oapi" ) -// ListImages lists all images func (s *ApiService) ListImages(ctx context.Context, request oapi.ListImagesRequestObject) (oapi.ListImagesResponseObject, error) { log := logger.FromContext(ctx) - imgs, err := s.ImageManager.ListImages(ctx) + domainImages, err := s.ImageManager.ListImages(ctx) if err != nil { log.Error("failed to list images", "error", err) return oapi.ListImages500JSONResponse{ @@ -21,14 +20,23 @@ func (s *ApiService) ListImages(ctx context.Context, request oapi.ListImagesRequ Message: "failed to list images", }, nil } - return oapi.ListImages200JSONResponse(imgs), nil + + oapiImages := make([]oapi.Image, len(domainImages)) + for i, img := range domainImages { + oapiImages[i] = imageToOAPI(img) + } + + return oapi.ListImages200JSONResponse(oapiImages), nil } -// CreateImage creates a new image from an OCI reference func (s *ApiService) CreateImage(ctx context.Context, request oapi.CreateImageRequestObject) (oapi.CreateImageResponseObject, error) { log := logger.FromContext(ctx) - img, err := s.ImageManager.CreateImage(ctx, *request.Body) + domainReq := images.CreateImageRequest{ + Name: request.Body.Name, + } + + img, err := s.ImageManager.CreateImage(ctx, domainReq) if err != nil { switch { case errors.Is(err, images.ErrInvalidName): @@ -49,7 +57,7 @@ func (s *ApiService) CreateImage(ctx context.Context, request oapi.CreateImageRe }, nil } } - return oapi.CreateImage202JSONResponse(*img), nil + return oapi.CreateImage202JSONResponse(imageToOAPI(*img)), nil } func (s *ApiService) GetImage(ctx context.Context, request oapi.GetImageRequestObject) (oapi.GetImageResponseObject, error) { @@ -71,7 +79,7 @@ func (s *ApiService) GetImage(ctx context.Context, request oapi.GetImageRequestO }, nil } } - return oapi.GetImage200JSONResponse(*img), nil + return oapi.GetImage200JSONResponse(imageToOAPI(*img)), nil } func (s *ApiService) DeleteImage(ctx context.Context, request oapi.DeleteImageRequestObject) (oapi.DeleteImageResponseObject, error) { @@ -96,3 +104,28 @@ func (s *ApiService) DeleteImage(ctx context.Context, request oapi.DeleteImageRe return oapi.DeleteImage204Response{}, nil } +func imageToOAPI(img images.Image) oapi.Image { + oapiImg := oapi.Image{ + Name: img.Name, + Status: oapi.ImageStatus(img.Status), + QueuePosition: img.QueuePosition, + Error: img.Error, + SizeBytes: img.SizeBytes, + CreatedAt: img.CreatedAt, + } + + if len(img.Entrypoint) > 0 { + oapiImg.Entrypoint = &img.Entrypoint + } + if len(img.Cmd) > 0 { + oapiImg.Cmd = &img.Cmd + } + if len(img.Env) > 0 { + oapiImg.Env = &img.Env + } + if img.WorkingDir != "" { + oapiImg.WorkingDir = &img.WorkingDir + } + + return oapiImg +} \ No newline at end of file diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 82976bcb..31048d7c 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -14,7 +14,7 @@ import ( func (s *ApiService) ListInstances(ctx context.Context, request oapi.ListInstancesRequestObject) (oapi.ListInstancesResponseObject, error) { log := logger.FromContext(ctx) - insts, err := s.InstanceManager.ListInstances(ctx) + domainInsts, err := s.InstanceManager.ListInstances(ctx) if err != nil { log.Error("failed to list instances", "error", err) return oapi.ListInstances500JSONResponse{ @@ -22,22 +22,34 @@ func (s *ApiService) ListInstances(ctx context.Context, request oapi.ListInstanc Message: "failed to list instances", }, nil } - return oapi.ListInstances200JSONResponse(insts), nil + + oapiInsts := make([]oapi.Instance, len(domainInsts)) + for i, inst := range domainInsts { + oapiInsts[i] = instanceToOAPI(inst) + } + + return oapi.ListInstances200JSONResponse(oapiInsts), nil } // CreateInstance creates and starts a new instance func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInstanceRequestObject) (oapi.CreateInstanceResponseObject, error) { log := logger.FromContext(ctx) - inst, err := s.InstanceManager.CreateInstance(ctx, *request.Body) + domainReq := instances.CreateInstanceRequest{ + Id: request.Body.Id, + Name: request.Body.Name, + Image: request.Body.Image, + } + + inst, err := s.InstanceManager.CreateInstance(ctx, domainReq) if err != nil { - log.Error("failed to create instance", "error", err, "name", request.Body.Name) + log.Error("failed to create instance", "error", err, "image", request.Body.Image) return oapi.CreateInstance500JSONResponse{ Code: "internal_error", Message: "failed to create instance", }, nil } - return oapi.CreateInstance201JSONResponse(*inst), nil + return oapi.CreateInstance201JSONResponse(instanceToOAPI(*inst)), nil } // GetInstance gets instance details @@ -60,9 +72,11 @@ func (s *ApiService) GetInstance(ctx context.Context, request oapi.GetInstanceRe }, nil } } - return oapi.GetInstance200JSONResponse(*inst), nil + return oapi.GetInstance200JSONResponse(instanceToOAPI(*inst)), nil } + + // DeleteInstance stops and deletes an instance func (s *ApiService) DeleteInstance(ctx context.Context, request oapi.DeleteInstanceRequestObject) (oapi.DeleteInstanceResponseObject, error) { log := logger.FromContext(ctx) @@ -111,7 +125,7 @@ func (s *ApiService) StandbyInstance(ctx context.Context, request oapi.StandbyIn }, nil } } - return oapi.StandbyInstance200JSONResponse(*inst), nil + return oapi.StandbyInstance200JSONResponse(instanceToOAPI(*inst)), nil } // RestoreInstance restores an instance from standby @@ -139,7 +153,7 @@ func (s *ApiService) RestoreInstance(ctx context.Context, request oapi.RestoreIn }, nil } } - return oapi.RestoreInstance200JSONResponse(*inst), nil + return oapi.RestoreInstance200JSONResponse(instanceToOAPI(*inst)), nil } // GetInstanceLogs streams instance logs @@ -182,7 +196,11 @@ func (s *ApiService) GetInstanceLogs(ctx context.Context, request oapi.GetInstan func (s *ApiService) AttachVolume(ctx context.Context, request oapi.AttachVolumeRequestObject) (oapi.AttachVolumeResponseObject, error) { log := logger.FromContext(ctx) - inst, err := s.InstanceManager.AttachVolume(ctx, request.Id, request.VolumeId, *request.Body) + domainReq := instances.AttachVolumeRequest{ + MountPath: request.Body.MountPath, + } + + inst, err := s.InstanceManager.AttachVolume(ctx, request.Id, request.VolumeId, domainReq) if err != nil { switch { case errors.Is(err, instances.ErrNotFound): @@ -198,7 +216,7 @@ func (s *ApiService) AttachVolume(ctx context.Context, request oapi.AttachVolume }, nil } } - return oapi.AttachVolume200JSONResponse(*inst), nil + return oapi.AttachVolume200JSONResponse(instanceToOAPI(*inst)), nil } // DetachVolume detaches a volume from an instance @@ -221,6 +239,14 @@ func (s *ApiService) DetachVolume(ctx context.Context, request oapi.DetachVolume }, nil } } - return oapi.DetachVolume200JSONResponse(*inst), nil + return oapi.DetachVolume200JSONResponse(instanceToOAPI(*inst)), nil } +func instanceToOAPI(inst instances.Instance) oapi.Instance { + return oapi.Instance{ + Id: inst.Id, + Name: inst.Name, + Image: inst.Image, + CreatedAt: inst.CreatedAt, + } +} \ No newline at end of file diff --git a/cmd/api/api/volumes.go b/cmd/api/api/volumes.go index 81ba2d4f..b1285bed 100644 --- a/cmd/api/api/volumes.go +++ b/cmd/api/api/volumes.go @@ -13,7 +13,7 @@ import ( func (s *ApiService) ListVolumes(ctx context.Context, request oapi.ListVolumesRequestObject) (oapi.ListVolumesResponseObject, error) { log := logger.FromContext(ctx) - vols, err := s.VolumeManager.ListVolumes(ctx) + domainVols, err := s.VolumeManager.ListVolumes(ctx) if err != nil { log.Error("failed to list volumes", "error", err) return oapi.ListVolumes500JSONResponse{ @@ -21,14 +21,26 @@ func (s *ApiService) ListVolumes(ctx context.Context, request oapi.ListVolumesRe Message: "failed to list volumes", }, nil } - return oapi.ListVolumes200JSONResponse(vols), nil + + oapiVols := make([]oapi.Volume, len(domainVols)) + for i, vol := range domainVols { + oapiVols[i] = volumeToOAPI(vol) + } + + return oapi.ListVolumes200JSONResponse(oapiVols), nil } // CreateVolume creates a new volume func (s *ApiService) CreateVolume(ctx context.Context, request oapi.CreateVolumeRequestObject) (oapi.CreateVolumeResponseObject, error) { log := logger.FromContext(ctx) - vol, err := s.VolumeManager.CreateVolume(ctx, *request.Body) + domainReq := volumes.CreateVolumeRequest{ + Name: request.Body.Name, + SizeGb: request.Body.SizeGb, + Id: request.Body.Id, + } + + vol, err := s.VolumeManager.CreateVolume(ctx, domainReq) if err != nil { log.Error("failed to create volume", "error", err, "name", request.Body.Name) return oapi.CreateVolume500JSONResponse{ @@ -36,7 +48,7 @@ func (s *ApiService) CreateVolume(ctx context.Context, request oapi.CreateVolume Message: "failed to create volume", }, nil } - return oapi.CreateVolume201JSONResponse(*vol), nil + return oapi.CreateVolume201JSONResponse(volumeToOAPI(*vol)), nil } // GetVolume gets volume details @@ -59,7 +71,7 @@ func (s *ApiService) GetVolume(ctx context.Context, request oapi.GetVolumeReques }, nil } } - return oapi.GetVolume200JSONResponse(*vol), nil + return oapi.GetVolume200JSONResponse(volumeToOAPI(*vol)), nil } // DeleteVolume deletes a volume @@ -90,3 +102,12 @@ func (s *ApiService) DeleteVolume(ctx context.Context, request oapi.DeleteVolume return oapi.DeleteVolume204Response{}, nil } +func volumeToOAPI(vol volumes.Volume) oapi.Volume { + return oapi.Volume{ + Id: vol.Id, + Name: vol.Name, + SizeGb: vol.SizeGb, + CreatedAt: vol.CreatedAt, + } +} + diff --git a/lib/images/manager.go b/lib/images/manager.go index 69c63407..1d117695 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -9,7 +9,6 @@ import ( "time" "github.com/distribution/reference" - "github.com/onkernel/hypeman/lib/oapi" ) const ( @@ -20,12 +19,11 @@ const ( StatusFailed = "failed" ) -// Manager handles image lifecycle operations type Manager interface { - ListImages(ctx context.Context) ([]oapi.Image, error) - CreateImage(ctx context.Context, req oapi.CreateImageRequest) (*oapi.Image, error) - GetImage(ctx context.Context, id string) (*oapi.Image, error) - DeleteImage(ctx context.Context, id string) error + ListImages(ctx context.Context) ([]Image, error) + CreateImage(ctx context.Context, req CreateImageRequest) (*Image, error) + GetImage(ctx context.Context, name string) (*Image, error) + DeleteImage(ctx context.Context, name string) error RecoverInterruptedBuilds() } @@ -46,21 +44,21 @@ func NewManager(dataDir string, ociClient *OCIClient, maxConcurrentBuilds int) M return m } -func (m *manager) ListImages(ctx context.Context) ([]oapi.Image, error) { +func (m *manager) ListImages(ctx context.Context) ([]Image, error) { metas, err := listMetadata(m.dataDir) if err != nil { return nil, fmt.Errorf("list metadata: %w", err) } - images := make([]oapi.Image, 0, len(metas)) + images := make([]Image, 0, len(metas)) for _, meta := range metas { - images = append(images, *meta.toOAPI()) + images = append(images, *meta.toImage()) } return images, nil } -func (m *manager) CreateImage(ctx context.Context, req oapi.CreateImageRequest) (*oapi.Image, error) { +func (m *manager) CreateImage(ctx context.Context, req CreateImageRequest) (*Image, error) { normalizedName, err := normalizeImageName(req.Name) if err != nil { return nil, fmt.Errorf("%w: %s", ErrInvalidName, err.Error()) @@ -85,14 +83,14 @@ func (m *manager) CreateImage(ctx context.Context, req oapi.CreateImageRequest) m.buildImage(context.Background(), normalizedName, req) }) - img := meta.toOAPI() + img := meta.toImage() if queuePos > 0 { img.QueuePosition = &queuePos } return img, nil } -func (m *manager) buildImage(ctx context.Context, imageName string, req oapi.CreateImageRequest) { +func (m *manager) buildImage(ctx context.Context, imageName string, req CreateImageRequest) { defer m.queue.MarkComplete(imageName) buildDir := filepath.Join(imageDir(m.dataDir, imageName), ".build") @@ -184,7 +182,7 @@ func (m *manager) RecoverInterruptedBuilds() { } } -func (m *manager) GetImage(ctx context.Context, name string) (*oapi.Image, error) { +func (m *manager) GetImage(ctx context.Context, name string) (*Image, error) { normalizedName, err := normalizeImageName(name) if err != nil { return nil, fmt.Errorf("%w: %s", ErrInvalidName, err.Error()) @@ -195,7 +193,7 @@ func (m *manager) GetImage(ctx context.Context, name string) (*oapi.Image, error return nil, err } - img := meta.toOAPI() + img := meta.toImage() if meta.Status == StatusPending { img.QueuePosition = m.queue.GetPosition(normalizedName) diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go index 18d69d78..8a36b7d7 100644 --- a/lib/images/manager_test.go +++ b/lib/images/manager_test.go @@ -7,7 +7,6 @@ import ( "testing" "time" - "github.com/onkernel/hypeman/lib/oapi" "github.com/stretchr/testify/require" ) @@ -19,7 +18,7 @@ func TestCreateImage(t *testing.T) { mgr := NewManager(dataDir, ociClient, 1) ctx := context.Background() - req := oapi.CreateImageRequest{ + req := CreateImageRequest{ Name: "docker.io/library/alpine:latest", } @@ -32,7 +31,7 @@ func TestCreateImage(t *testing.T) { img, err = mgr.GetImage(ctx, img.Name) require.NoError(t, err) - require.Equal(t, oapi.ImageStatus(StatusReady), img.Status) + require.Equal(t, StatusReady, img.Status) require.NotNil(t, img.SizeBytes) require.Greater(t, *img.SizeBytes, int64(0)) @@ -49,7 +48,7 @@ func TestCreateImageDifferentTag(t *testing.T) { mgr := NewManager(dataDir, ociClient, 1) ctx := context.Background() - req := oapi.CreateImageRequest{ + req := CreateImageRequest{ Name: "docker.io/library/alpine:3.18", } @@ -69,11 +68,10 @@ func TestCreateImageDuplicate(t *testing.T) { mgr := NewManager(dataDir, ociClient, 1) ctx := context.Background() - req := oapi.CreateImageRequest{ + req := CreateImageRequest{ Name: "docker.io/library/alpine:latest", } - // Create first image img1, err := mgr.CreateImage(ctx, req) require.NoError(t, err) @@ -98,8 +96,7 @@ func TestListImages(t *testing.T) { require.NoError(t, err) require.Len(t, images, 0) - // Create first image - req1 := oapi.CreateImageRequest{ + req1 := CreateImageRequest{ Name: "docker.io/library/alpine:latest", } img1, err := mgr.CreateImage(ctx, req1) @@ -112,7 +109,7 @@ func TestListImages(t *testing.T) { require.NoError(t, err) require.Len(t, images, 1) require.Equal(t, "docker.io/library/alpine:latest", images[0].Name) - require.Equal(t, oapi.ImageStatus(StatusReady), images[0].Status) + require.Equal(t, StatusReady, images[0].Status) } func TestGetImage(t *testing.T) { @@ -123,7 +120,7 @@ func TestGetImage(t *testing.T) { mgr := NewManager(dataDir, ociClient, 1) ctx := context.Background() - req := oapi.CreateImageRequest{ + req := CreateImageRequest{ Name: "docker.io/library/alpine:latest", } @@ -136,7 +133,7 @@ func TestGetImage(t *testing.T) { require.NoError(t, err) require.NotNil(t, img) require.Equal(t, created.Name, img.Name) - require.Equal(t, oapi.ImageStatus(StatusReady), img.Status) + require.Equal(t, StatusReady, img.Status) require.NotNil(t, img.SizeBytes) } @@ -161,7 +158,7 @@ func TestDeleteImage(t *testing.T) { mgr := NewManager(dataDir, ociClient, 1) ctx := context.Background() - req := oapi.CreateImageRequest{ + req := CreateImageRequest{ Name: "docker.io/library/alpine:latest", } @@ -227,11 +224,11 @@ func waitForReady(t *testing.T, mgr Manager, ctx context.Context, imageName stri t.Logf("Status: %s", img.Status) } - if img.Status == oapi.ImageStatus(StatusReady) { + if img.Status == StatusReady { return } - if img.Status == oapi.ImageStatus(StatusFailed) { + if img.Status == StatusFailed { errMsg := "" if img.Error != nil { errMsg = *img.Error diff --git a/lib/images/queue.go b/lib/images/queue.go index aae65684..e3b8f8ba 100644 --- a/lib/images/queue.go +++ b/lib/images/queue.go @@ -1,14 +1,10 @@ package images -import ( - "sync" - - "github.com/onkernel/hypeman/lib/oapi" -) +import "sync" type QueuedBuild struct { ImageName string - Request oapi.CreateImageRequest + Request CreateImageRequest StartFn func() } @@ -31,7 +27,7 @@ func NewBuildQueue(maxConcurrent int) *BuildQueue { } } -func (q *BuildQueue) Enqueue(imageName string, req oapi.CreateImageRequest, startFn func()) int { +func (q *BuildQueue) Enqueue(imageName string, req CreateImageRequest, startFn func()) int { q.mu.Lock() defer q.mu.Unlock() diff --git a/lib/images/storage.go b/lib/images/storage.go index 5b0b3cf8..a3d14dd2 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -7,27 +7,25 @@ import ( "path/filepath" "strings" "time" - - "github.com/onkernel/hypeman/lib/oapi" ) type imageMetadata struct { - Name string `json:"name"` - Status string `json:"status"` - Error *string `json:"error,omitempty"` - Request *oapi.CreateImageRequest `json:"request,omitempty"` - SizeBytes int64 `json:"size_bytes"` - Entrypoint []string `json:"entrypoint,omitempty"` - Cmd []string `json:"cmd,omitempty"` - Env map[string]string `json:"env,omitempty"` - WorkingDir string `json:"working_dir,omitempty"` - CreatedAt time.Time `json:"created_at"` + Name string `json:"name"` + Status string `json:"status"` + Error *string `json:"error,omitempty"` + Request *CreateImageRequest `json:"request,omitempty"` + SizeBytes int64 `json:"size_bytes"` + Entrypoint []string `json:"entrypoint,omitempty"` + Cmd []string `json:"cmd,omitempty"` + Env map[string]string `json:"env,omitempty"` + WorkingDir string `json:"working_dir,omitempty"` + CreatedAt time.Time `json:"created_at"` } -func (m *imageMetadata) toOAPI() *oapi.Image { - img := &oapi.Image{ +func (m *imageMetadata) toImage() *Image { + img := &Image{ Name: m.Name, - Status: oapi.ImageStatus(m.Status), + Status: m.Status, Error: m.Error, CreatedAt: m.CreatedAt, } @@ -38,16 +36,16 @@ func (m *imageMetadata) toOAPI() *oapi.Image { } if len(m.Entrypoint) > 0 { - img.Entrypoint = &m.Entrypoint + img.Entrypoint = m.Entrypoint } if len(m.Cmd) > 0 { - img.Cmd = &m.Cmd + img.Cmd = m.Cmd } if len(m.Env) > 0 { - img.Env = &m.Env + img.Env = m.Env } if m.WorkingDir != "" { - img.WorkingDir = &m.WorkingDir + img.WorkingDir = m.WorkingDir } return img diff --git a/lib/images/types.go b/lib/images/types.go new file mode 100644 index 00000000..6e5fcd55 --- /dev/null +++ b/lib/images/types.go @@ -0,0 +1,23 @@ +package images + +import "time" + +// Image represents a container image converted to bootable disk +type Image struct { + Name string + Status string + QueuePosition *int + Error *string + SizeBytes *int64 + Entrypoint []string + Cmd []string + Env map[string]string + WorkingDir string + CreatedAt time.Time +} + +// CreateImageRequest represents a request to create an image +type CreateImageRequest struct { + Name string +} + diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 58978307..bfc6abf2 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -3,102 +3,62 @@ package instances import ( "context" "fmt" - - "github.com/onkernel/hypeman/lib/oapi" ) -// Manager handles instance lifecycle operations type Manager interface { - ListInstances(ctx context.Context) ([]oapi.Instance, error) - CreateInstance(ctx context.Context, req oapi.CreateInstanceRequest) (*oapi.Instance, error) - GetInstance(ctx context.Context, id string) (*oapi.Instance, error) + ListInstances(ctx context.Context) ([]Instance, error) + CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) + GetInstance(ctx context.Context, id string) (*Instance, error) DeleteInstance(ctx context.Context, id string) error - StandbyInstance(ctx context.Context, id string) (*oapi.Instance, error) - RestoreInstance(ctx context.Context, id string) (*oapi.Instance, error) + StandbyInstance(ctx context.Context, id string) (*Instance, error) + RestoreInstance(ctx context.Context, id string) (*Instance, error) GetInstanceLogs(ctx context.Context, id string, follow bool, tail int) (string, error) - AttachVolume(ctx context.Context, id string, volumeId string, req oapi.AttachVolumeRequest) (*oapi.Instance, error) - DetachVolume(ctx context.Context, id string, volumeId string) (*oapi.Instance, error) + AttachVolume(ctx context.Context, id string, volumeId string, req AttachVolumeRequest) (*Instance, error) + DetachVolume(ctx context.Context, id string, volumeId string) (*Instance, error) } type manager struct { dataDir string } -// NewManager creates a new instance manager func NewManager(dataDir string) Manager { return &manager{ dataDir: dataDir, } } -func (m *manager) ListInstances(ctx context.Context) ([]oapi.Instance, error) { - // TODO: implement - return []oapi.Instance{}, nil +func (m *manager) ListInstances(ctx context.Context) ([]Instance, error) { + return []Instance{}, nil } -func (m *manager) CreateInstance(ctx context.Context, req oapi.CreateInstanceRequest) (*oapi.Instance, error) { - // TODO: implement +func (m *manager) CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) { return nil, fmt.Errorf("instance creation not yet implemented") } -func (m *manager) GetInstance(ctx context.Context, id string) (*oapi.Instance, error) { - // TODO: implement actual logic - exists := false - if !exists { - return nil, ErrNotFound - } - return nil, fmt.Errorf("get instance not yet implemented") +func (m *manager) GetInstance(ctx context.Context, id string) (*Instance, error) { + return nil, ErrNotFound } func (m *manager) DeleteInstance(ctx context.Context, id string) error { - // TODO: implement actual logic - exists := false - if !exists { - return ErrNotFound - } - return fmt.Errorf("delete instance not yet implemented") + return ErrNotFound } -func (m *manager) StandbyInstance(ctx context.Context, id string) (*oapi.Instance, error) { - // TODO: implement actual logic - exists := false - if !exists { - return nil, ErrNotFound - } - // Check if instance is in correct state (e.g., Running) - validState := false - if !validState { - return nil, ErrInvalidState - } +func (m *manager) StandbyInstance(ctx context.Context, id string) (*Instance, error) { return nil, fmt.Errorf("standby instance not yet implemented") } -func (m *manager) RestoreInstance(ctx context.Context, id string) (*oapi.Instance, error) { - // TODO: implement actual logic - exists := false - if !exists { - return nil, ErrNotFound - } - // Check if instance is in Standby state - inStandby := false - if !inStandby { - return nil, ErrInvalidState - } +func (m *manager) RestoreInstance(ctx context.Context, id string) (*Instance, error) { return nil, fmt.Errorf("restore instance not yet implemented") } func (m *manager) GetInstanceLogs(ctx context.Context, id string, follow bool, tail int) (string, error) { - // TODO: implement return "", fmt.Errorf("get instance logs not yet implemented") } -func (m *manager) AttachVolume(ctx context.Context, id string, volumeId string, req oapi.AttachVolumeRequest) (*oapi.Instance, error) { - // TODO: implement +func (m *manager) AttachVolume(ctx context.Context, id string, volumeId string, req AttachVolumeRequest) (*Instance, error) { return nil, fmt.Errorf("attach volume not yet implemented") } -func (m *manager) DetachVolume(ctx context.Context, id string, volumeId string) (*oapi.Instance, error) { - // TODO: implement +func (m *manager) DetachVolume(ctx context.Context, id string, volumeId string) (*Instance, error) { return nil, fmt.Errorf("detach volume not yet implemented") } - diff --git a/lib/instances/types.go b/lib/instances/types.go new file mode 100644 index 00000000..8b118ca4 --- /dev/null +++ b/lib/instances/types.go @@ -0,0 +1,21 @@ +package instances + +import "time" + +type Instance struct { + Id string + Name string + Image string + CreatedAt time.Time +} + +type CreateInstanceRequest struct { + Id string + Name string + Image string +} + +type AttachVolumeRequest struct { + MountPath string +} + diff --git a/lib/volumes/manager.go b/lib/volumes/manager.go index c7d64eec..bf22e7bd 100644 --- a/lib/volumes/manager.go +++ b/lib/volumes/manager.go @@ -3,15 +3,12 @@ package volumes import ( "context" "fmt" - - "github.com/onkernel/hypeman/lib/oapi" ) -// Manager handles volume lifecycle operations type Manager interface { - ListVolumes(ctx context.Context) ([]oapi.Volume, error) - CreateVolume(ctx context.Context, req oapi.CreateVolumeRequest) (*oapi.Volume, error) - GetVolume(ctx context.Context, id string) (*oapi.Volume, error) + ListVolumes(ctx context.Context) ([]Volume, error) + CreateVolume(ctx context.Context, req CreateVolumeRequest) (*Volume, error) + GetVolume(ctx context.Context, id string) (*Volume, error) DeleteVolume(ctx context.Context, id string) error } @@ -19,43 +16,24 @@ type manager struct { dataDir string } -// NewManager creates a new volume manager func NewManager(dataDir string) Manager { return &manager{ dataDir: dataDir, } } -func (m *manager) ListVolumes(ctx context.Context) ([]oapi.Volume, error) { - // TODO: implement - return []oapi.Volume{}, nil +func (m *manager) ListVolumes(ctx context.Context) ([]Volume, error) { + return []Volume{}, nil } -func (m *manager) CreateVolume(ctx context.Context, req oapi.CreateVolumeRequest) (*oapi.Volume, error) { - // TODO: implement +func (m *manager) CreateVolume(ctx context.Context, req CreateVolumeRequest) (*Volume, error) { return nil, fmt.Errorf("volume creation not yet implemented") } -func (m *manager) GetVolume(ctx context.Context, id string) (*oapi.Volume, error) { - // TODO: implement actual logic - exists := false - if !exists { - return nil, ErrNotFound - } - return nil, fmt.Errorf("get volume not yet implemented") +func (m *manager) GetVolume(ctx context.Context, id string) (*Volume, error) { + return nil, ErrNotFound } func (m *manager) DeleteVolume(ctx context.Context, id string) error { - // TODO: implement actual logic - exists := false - if !exists { - return ErrNotFound - } - // Check if volume is attached to any instance - inUse := false - if inUse { - return ErrInUse - } - return fmt.Errorf("delete volume not yet implemented") + return ErrNotFound } - diff --git a/lib/volumes/types.go b/lib/volumes/types.go new file mode 100644 index 00000000..971bf967 --- /dev/null +++ b/lib/volumes/types.go @@ -0,0 +1,17 @@ +package volumes + +import "time" + +type Volume struct { + Id string + Name string + SizeGb int + CreatedAt time.Time +} + +type CreateVolumeRequest struct { + Name string + SizeGb int + Id *string +} + From a5737abaf84ad45b22d94392ef94d450cc2b6dde Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 10:59:34 -0500 Subject: [PATCH 23/37] Idempotency and race conditions --- cmd/api/api/images.go | 5 -- cmd/api/api/images_test.go | 93 ++++++++++++++++++++++++++++++++++++++ lib/images/errors.go | 5 +- lib/images/manager.go | 20 ++++++-- lib/images/manager_test.go | 9 ++-- lib/images/queue.go | 24 +++++++++- 6 files changed, 139 insertions(+), 17 deletions(-) diff --git a/cmd/api/api/images.go b/cmd/api/api/images.go index 55210c3f..4cc7cd93 100644 --- a/cmd/api/api/images.go +++ b/cmd/api/api/images.go @@ -44,11 +44,6 @@ func (s *ApiService) CreateImage(ctx context.Context, request oapi.CreateImageRe Code: "invalid_name", Message: err.Error(), }, nil - case errors.Is(err, images.ErrAlreadyExists): - return oapi.CreateImage400JSONResponse{ - Code: "already_exists", - Message: "image already exists", - }, nil default: log.Error("failed to create image", "error", err, "name", request.Body.Name) return oapi.CreateImage500JSONResponse{ diff --git a/cmd/api/api/images_test.go b/cmd/api/api/images_test.go index 38527c4c..2a181fe6 100644 --- a/cmd/api/api/images_test.go +++ b/cmd/api/api/images_test.go @@ -193,6 +193,99 @@ func TestCreateImage_InvalidName(t *testing.T) { } } +func TestCreateImage_Idempotent(t *testing.T) { + svc := newTestService(t) + ctx := ctx() + + // Create first image to occupy queue position 0 + t.Log("Creating first image (busybox) to occupy queue...") + _, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ + Body: &oapi.CreateImageRequest{Name: "docker.io/library/busybox:latest"}, + }) + require.NoError(t, err) + + imageName := "docker.io/library/alpine:3.18" + + // First call - should create and queue at position 1 + t.Log("First CreateImage call (alpine)...") + resp1, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ + Body: &oapi.CreateImageRequest{Name: imageName}, + }) + require.NoError(t, err) + + accepted1, ok := resp1.(oapi.CreateImage202JSONResponse) + require.True(t, ok, "expected 202 response") + img1 := oapi.Image(accepted1) + require.Equal(t, imageName, img1.Name) + require.Equal(t, oapi.ImageStatus(images.StatusPending), img1.Status) + require.NotNil(t, img1.QueuePosition, "should have queue position") + require.Equal(t, 1, *img1.QueuePosition, "should be at position 1") + t.Logf("First call: status=%s, queue_position=%v", img1.Status, formatQueuePos(img1.QueuePosition)) + + // Second call immediately - should return existing with same queue position + t.Log("Second CreateImage call (immediate duplicate)...") + resp2, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ + Body: &oapi.CreateImageRequest{Name: imageName}, + }) + require.NoError(t, err) + + accepted2, ok := resp2.(oapi.CreateImage202JSONResponse) + require.True(t, ok, "expected 202 response") + img2 := oapi.Image(accepted2) + require.Equal(t, imageName, img2.Name) + require.Equal(t, oapi.ImageStatus(images.StatusPending), img2.Status) + require.NotNil(t, img2.QueuePosition, "should have queue position") + require.Equal(t, 1, *img2.QueuePosition, "should still be at position 1") + t.Logf("Second call: status=%s, queue_position=%v", img2.Status, formatQueuePos(img2.QueuePosition)) + + // Wait for build to complete + t.Log("Waiting for build to complete...") + for i := 0; i < 3000; i++ { + getResp, err := svc.GetImage(ctx, oapi.GetImageRequestObject{Name: imageName}) + require.NoError(t, err) + + imgResp, ok := getResp.(oapi.GetImage200JSONResponse) + require.True(t, ok, "expected 200 response") + + currentImg := oapi.Image(imgResp) + + if currentImg.Status == oapi.ImageStatus(images.StatusReady) { + t.Log("Build complete!") + break + } + + if currentImg.Status == oapi.ImageStatus(images.StatusFailed) { + errMsg := "" + if currentImg.Error != nil { + errMsg = *currentImg.Error + } + t.Fatalf("Build failed: %s", errMsg) + } + + time.Sleep(10 * time.Millisecond) + } + + // Third call after completion - should return ready image with no queue position + t.Log("Third CreateImage call (after completion)...") + resp3, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ + Body: &oapi.CreateImageRequest{Name: imageName}, + }) + require.NoError(t, err) + + accepted3, ok := resp3.(oapi.CreateImage202JSONResponse) + require.True(t, ok, "expected 202 response") + img3 := oapi.Image(accepted3) + require.Equal(t, imageName, img3.Name) + require.Equal(t, oapi.ImageStatus(images.StatusReady), img3.Status, "should return ready image") + require.Nil(t, img3.QueuePosition, "ready image should have no queue position") + require.NotNil(t, img3.SizeBytes) + require.Greater(t, *img3.SizeBytes, int64(0)) + t.Logf("Third call: status=%s, queue_position=%v, size=%d", + img3.Status, formatQueuePos(img3.QueuePosition), *img3.SizeBytes) + + t.Log("Idempotency test passed!") +} + func getQueuePos(pos *int) int { if pos == nil { return 0 diff --git a/lib/images/errors.go b/lib/images/errors.go index acabf3bd..63b3bb3b 100644 --- a/lib/images/errors.go +++ b/lib/images/errors.go @@ -3,7 +3,6 @@ package images import "errors" var ( - ErrNotFound = errors.New("image not found") - ErrAlreadyExists = errors.New("image already exists") - ErrInvalidName = errors.New("invalid image name") + ErrNotFound = errors.New("image not found") + ErrInvalidName = errors.New("invalid image name") ) diff --git a/lib/images/manager.go b/lib/images/manager.go index 1d117695..9855cfe3 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "sort" + "sync" "time" "github.com/distribution/reference" @@ -28,9 +29,10 @@ type Manager interface { } type manager struct { - dataDir string + dataDir string ociClient *OCIClient queue *BuildQueue + createMu sync.Mutex } // NewManager creates a new image manager with OCI client @@ -64,10 +66,20 @@ func (m *manager) CreateImage(ctx context.Context, req CreateImageRequest) (*Ima return nil, fmt.Errorf("%w: %s", ErrInvalidName, err.Error()) } - if imageExists(m.dataDir, normalizedName) { - return nil, ErrAlreadyExists + m.createMu.Lock() + defer m.createMu.Unlock() + + // Check if already exists (handles ready, failed, and in-progress) + if meta, err := readMetadata(m.dataDir, normalizedName); err == nil { + img := meta.toImage() + // Add dynamic queue position only for pending status + if meta.Status == StatusPending { + img.QueuePosition = m.queue.GetPosition(normalizedName) + } + return img, nil } + // Create metadata (we know it doesn't exist) meta := &imageMetadata{ Name: normalizedName, Status: StatusPending, @@ -79,6 +91,7 @@ func (m *manager) CreateImage(ctx context.Context, req CreateImageRequest) (*Ima return nil, fmt.Errorf("write initial metadata: %w", err) } + // Enqueue (we know it's not already queued due to mutex) queuePos := m.queue.Enqueue(normalizedName, req, func() { m.buildImage(context.Background(), normalizedName, req) }) @@ -91,7 +104,6 @@ func (m *manager) CreateImage(ctx context.Context, req CreateImageRequest) (*Ima } func (m *manager) buildImage(ctx context.Context, imageName string, req CreateImageRequest) { - defer m.queue.MarkComplete(imageName) buildDir := filepath.Join(imageDir(m.dataDir, imageName), ".build") tempDir := filepath.Join(buildDir, "rootfs") diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go index 8a36b7d7..d44dfed8 100644 --- a/lib/images/manager_test.go +++ b/lib/images/manager_test.go @@ -77,9 +77,12 @@ func TestCreateImageDuplicate(t *testing.T) { waitForReady(t, mgr, ctx, img1.Name) - // Try to create duplicate - _, err = mgr.CreateImage(ctx, req) - require.ErrorIs(t, err, ErrAlreadyExists) + // Second create should be idempotent and return existing image + img2, err := mgr.CreateImage(ctx, req) + require.NoError(t, err) + require.NotNil(t, img2) + require.Equal(t, img1.Name, img2.Name) + require.Equal(t, StatusReady, img2.Status) } func TestListImages(t *testing.T) { diff --git a/lib/images/queue.go b/lib/images/queue.go index e3b8f8ba..115d267c 100644 --- a/lib/images/queue.go +++ b/lib/images/queue.go @@ -27,19 +27,39 @@ func NewBuildQueue(maxConcurrent int) *BuildQueue { } } +// Enqueue adds a build to the queue. Returns queue position (0 if started immediately, >0 if queued). +// If the image is already building or queued, returns its current position without re-enqueueing. func (q *BuildQueue) Enqueue(imageName string, req CreateImageRequest, startFn func()) int { q.mu.Lock() defer q.mu.Unlock() + // Check if already building (position 0, actively running) + if q.active[imageName] { + return 0 + } + + // Check if already in pending queue + for i, build := range q.pending { + if build.ImageName == imageName { + return i + 1 // Return existing queue position + } + } + + // Wrap the function to auto-complete + wrappedFn := func() { + defer q.MarkComplete(imageName) + startFn() + } + build := QueuedBuild{ ImageName: imageName, Request: req, - StartFn: startFn, + StartFn: wrappedFn, } if len(q.active) < q.maxConcurrent { q.active[imageName] = true - go startFn() + go wrappedFn() return 0 } From 3e8e83572efa51718e88f74324a6dc3c87459743 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 11:01:53 -0500 Subject: [PATCH 24/37] Add erofs to ci --- .github/workflows/test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 10944ca1..e828044d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,10 @@ jobs: go-version: '1.25' - name: Install dependencies - run: go mod download + run: | + set -xe + sudo apt-get install -y erofs-utils + go mod download # Avoids rate limits when running the tests # Tests includes pulling, then converting to disk images From 3c9fc0d56c9351aa13acef02dd8050de2abfcce3 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 11:09:14 -0500 Subject: [PATCH 25/37] OCI client internal to image manager --- cmd/api/api/api_test.go | 7 +++--- cmd/api/wire.go | 1 - cmd/api/wire_gen.go | 3 +-- go.sum | 6 +++++ lib/images/manager.go | 15 ++++++++---- lib/images/manager_test.go | 48 +++++++++++++------------------------- lib/images/oci.go | 23 +++++++----------- lib/providers/providers.go | 11 ++------- 8 files changed, 48 insertions(+), 66 deletions(-) diff --git a/cmd/api/api/api_test.go b/cmd/api/api/api_test.go index c1ac23de..5afad7ed 100644 --- a/cmd/api/api/api_test.go +++ b/cmd/api/api/api_test.go @@ -16,15 +16,14 @@ func newTestService(t *testing.T) *ApiService { DataDir: t.TempDir(), } - // Create OCI client for testing - ociClient, err := images.NewOCIClient(cfg.DataDir + "/system/oci-cache") + imageMgr, err := images.NewManager(cfg.DataDir, 1) if err != nil { - t.Fatalf("failed to create OCI client: %v", err) + t.Fatalf("failed to create image manager: %v", err) } return &ApiService{ Config: cfg, - ImageManager: images.NewManager(cfg.DataDir, ociClient, 1), + ImageManager: imageMgr, InstanceManager: instances.NewManager(cfg.DataDir), VolumeManager: volumes.NewManager(cfg.DataDir), } diff --git a/cmd/api/wire.go b/cmd/api/wire.go index 174e6fd5..f9678ba6 100644 --- a/cmd/api/wire.go +++ b/cmd/api/wire.go @@ -32,7 +32,6 @@ func initializeApp() (*application, func(), error) { providers.ProvideLogger, providers.ProvideContext, providers.ProvideConfig, - providers.ProvideOCIClient, providers.ProvideImageManager, providers.ProvideInstanceManager, providers.ProvideVolumeManager, diff --git a/cmd/api/wire_gen.go b/cmd/api/wire_gen.go index aa06a69f..e254a4a6 100644 --- a/cmd/api/wire_gen.go +++ b/cmd/api/wire_gen.go @@ -28,11 +28,10 @@ func initializeApp() (*application, func(), error) { logger := providers.ProvideLogger() context := providers.ProvideContext(logger) config := providers.ProvideConfig() - ociClient, err := providers.ProvideOCIClient(config) + manager, err := providers.ProvideImageManager(config) if err != nil { return nil, nil, err } - manager := providers.ProvideImageManager(config, ociClient) instancesManager := providers.ProvideInstanceManager(config) volumesManager := providers.ProvideVolumeManager(config) apiService := api.New(config, manager, instancesManager, volumesManager) diff --git a/go.sum b/go.sum index 107c4ce9..39ecf526 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,8 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/go-containerregistry v0.20.3 h1:oNx7IdTI936V8CQRveCjaxOiegWwvM7kqkbXTpyiovI= github.com/google/go-containerregistry v0.20.3/go.mod h1:w00pIgBRDVUDFM6bq+Qx8lwNWK+cxgCuX1vd3PIBDNI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -282,6 +284,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -350,6 +354,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= diff --git a/lib/images/manager.go b/lib/images/manager.go index 9855cfe3..8f8a2a2e 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -30,20 +30,27 @@ type Manager interface { type manager struct { dataDir string - ociClient *OCIClient + ociClient *ociClient queue *BuildQueue createMu sync.Mutex } -// NewManager creates a new image manager with OCI client -func NewManager(dataDir string, ociClient *OCIClient, maxConcurrentBuilds int) Manager { +// NewManager creates a new image manager +func NewManager(dataDir string, maxConcurrentBuilds int) (Manager, error) { + // Create cache directory under dataDir for OCI layouts + cacheDir := filepath.Join(dataDir, "system", "oci-cache") + ociClient, err := newOCIClient(cacheDir) + if err != nil { + return nil, fmt.Errorf("create oci client: %w", err) + } + m := &manager{ dataDir: dataDir, ociClient: ociClient, queue: NewBuildQueue(maxConcurrentBuilds), } m.RecoverInterruptedBuilds() - return m + return m, nil } func (m *manager) ListImages(ctx context.Context) ([]Image, error) { diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go index d44dfed8..eaccc5c5 100644 --- a/lib/images/manager_test.go +++ b/lib/images/manager_test.go @@ -11,11 +11,9 @@ import ( ) func TestCreateImage(t *testing.T) { - ociClient, err := NewOCIClient(t.TempDir()) - require.NoError(t, err) - dataDir := t.TempDir() - mgr := NewManager(dataDir, ociClient, 1) + mgr, err := NewManager(dataDir, 1) + require.NoError(t, err) ctx := context.Background() req := CreateImageRequest{ @@ -41,11 +39,9 @@ func TestCreateImage(t *testing.T) { } func TestCreateImageDifferentTag(t *testing.T) { - ociClient, err := NewOCIClient(t.TempDir()) - require.NoError(t, err) - dataDir := t.TempDir() - mgr := NewManager(dataDir, ociClient, 1) + mgr, err := NewManager(dataDir, 1) + require.NoError(t, err) ctx := context.Background() req := CreateImageRequest{ @@ -61,11 +57,9 @@ func TestCreateImageDifferentTag(t *testing.T) { } func TestCreateImageDuplicate(t *testing.T) { - ociClient, err := NewOCIClient(t.TempDir()) - require.NoError(t, err) - dataDir := t.TempDir() - mgr := NewManager(dataDir, ociClient, 1) + mgr, err := NewManager(dataDir, 1) + require.NoError(t, err) ctx := context.Background() req := CreateImageRequest{ @@ -86,11 +80,9 @@ func TestCreateImageDuplicate(t *testing.T) { } func TestListImages(t *testing.T) { - ociClient, err := NewOCIClient(t.TempDir()) - require.NoError(t, err) - dataDir := t.TempDir() - mgr := NewManager(dataDir, ociClient, 1) + mgr, err := NewManager(dataDir, 1) + require.NoError(t, err) ctx := context.Background() @@ -116,11 +108,9 @@ func TestListImages(t *testing.T) { } func TestGetImage(t *testing.T) { - ociClient, err := NewOCIClient(t.TempDir()) - require.NoError(t, err) - dataDir := t.TempDir() - mgr := NewManager(dataDir, ociClient, 1) + mgr, err := NewManager(dataDir, 1) + require.NoError(t, err) ctx := context.Background() req := CreateImageRequest{ @@ -141,11 +131,9 @@ func TestGetImage(t *testing.T) { } func TestGetImageNotFound(t *testing.T) { - ociClient, err := NewOCIClient(t.TempDir()) - require.NoError(t, err) - dataDir := t.TempDir() - mgr := NewManager(dataDir, ociClient, 1) + mgr, err := NewManager(dataDir, 1) + require.NoError(t, err) ctx := context.Background() @@ -154,11 +142,9 @@ func TestGetImageNotFound(t *testing.T) { } func TestDeleteImage(t *testing.T) { - ociClient, err := NewOCIClient(t.TempDir()) - require.NoError(t, err) - dataDir := t.TempDir() - mgr := NewManager(dataDir, ociClient, 1) + mgr, err := NewManager(dataDir, 1) + require.NoError(t, err) ctx := context.Background() req := CreateImageRequest{ @@ -182,11 +168,9 @@ func TestDeleteImage(t *testing.T) { } func TestDeleteImageNotFound(t *testing.T) { - ociClient, err := NewOCIClient(t.TempDir()) - require.NoError(t, err) - dataDir := t.TempDir() - mgr := NewManager(dataDir, ociClient, 1) + mgr, err := NewManager(dataDir, 1) + require.NoError(t, err) ctx := context.Background() diff --git a/lib/images/oci.go b/lib/images/oci.go index c0a80f21..bdf7e6b4 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -17,25 +17,20 @@ import ( "github.com/opencontainers/umoci/oci/layer" ) -// OCIClient handles OCI image operations without requiring Docker daemon -type OCIClient struct { +// ociClient handles OCI image operations without requiring Docker daemon +type ociClient struct { cacheDir string } -// NewOCIClient creates a new OCI client -func NewOCIClient(cacheDir string) (*OCIClient, error) { +// newOCIClient creates a new OCI client +func newOCIClient(cacheDir string) (*ociClient, error) { if err := os.MkdirAll(cacheDir, 0755); err != nil { return nil, fmt.Errorf("create cache dir: %w", err) } - return &OCIClient{cacheDir: cacheDir}, nil + return &ociClient{cacheDir: cacheDir}, nil } -// Close closes the OCI client (no-op for now) -func (c *OCIClient) Close() error { - return nil -} - -func (c *OCIClient) pullAndExport(ctx context.Context, imageRef, exportDir string) (*containerMetadata, error) { +func (c *ociClient) pullAndExport(ctx context.Context, imageRef, exportDir string) (*containerMetadata, error) { // Use persistent OCI layout for caching (parse imageRef into path) ociLayoutDir := filepath.Join(c.cacheDir, imageNameToPath(imageRef)) if err := os.MkdirAll(ociLayoutDir, 0755); err != nil { @@ -58,7 +53,7 @@ func (c *OCIClient) pullAndExport(ctx context.Context, imageRef, exportDir strin return meta, nil } -func (c *OCIClient) pullToOCILayout(ctx context.Context, imageRef, ociLayoutDir string) error { +func (c *ociClient) pullToOCILayout(ctx context.Context, imageRef, ociLayoutDir string) error { // Parse source reference (docker://...) srcRef, err := docker.ParseReference("//" + imageRef) if err != nil { @@ -91,7 +86,7 @@ func (c *OCIClient) pullToOCILayout(ctx context.Context, imageRef, ociLayoutDir } // extractOCIMetadata reads metadata from OCI layout config.json -func (c *OCIClient) extractOCIMetadata(ociLayoutDir string) (*containerMetadata, error) { +func (c *ociClient) extractOCIMetadata(ociLayoutDir string) (*containerMetadata, error) { // Open the OCI layout casEngine, err := dir.Open(ociLayoutDir) if err != nil { @@ -159,7 +154,7 @@ func (c *OCIClient) extractOCIMetadata(ociLayoutDir string) (*containerMetadata, } // unpackLayers unpacks all OCI layers to a target directory using umoci -func (c *OCIClient) unpackLayers(ctx context.Context, ociLayoutDir, targetDir string) error { +func (c *ociClient) unpackLayers(ctx context.Context, ociLayoutDir, targetDir string) error { // Open OCI layout casEngine, err := dir.Open(ociLayoutDir) if err != nil { diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 33a785c0..d32f8f46 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -29,16 +29,9 @@ func ProvideConfig() *config.Config { return config.Load() } -// ProvideOCIClient provides an OCI client -func ProvideOCIClient(cfg *config.Config) (*images.OCIClient, error) { - // Use a cache directory under dataDir for OCI layouts - cacheDir := cfg.DataDir + "/system/oci-cache" - return images.NewOCIClient(cacheDir) -} - // ProvideImageManager provides the image manager -func ProvideImageManager(cfg *config.Config, ociClient *images.OCIClient) images.Manager { - return images.NewManager(cfg.DataDir, ociClient, cfg.MaxConcurrentBuilds) +func ProvideImageManager(cfg *config.Config) (images.Manager, error) { + return images.NewManager(cfg.DataDir, cfg.MaxConcurrentBuilds) } // ProvideInstanceManager provides the instance manager From 5fda8e9e2ae542c442564d4000738577f19342c7 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 13:29:20 -0500 Subject: [PATCH 26/37] Disambiguate digests and tags --- cmd/api/api/images.go | 1 + lib/images/README.md | 68 +++++++++-- lib/images/manager.go | 209 ++++++++++++++++++++++------------ lib/images/manager_test.go | 70 +++++++++--- lib/images/oci.go | 153 +++++++++++++++++++++---- lib/images/reference.go | 160 ++++++++++++++++++++++++++ lib/images/storage.go | 200 ++++++++++++++++++++++---------- lib/images/types.go | 3 +- lib/images/validation_test.go | 63 +++++++++- lib/oapi/oapi.go | 8 +- openapi.yaml | 13 +-- 11 files changed, 749 insertions(+), 199 deletions(-) create mode 100644 lib/images/reference.go diff --git a/cmd/api/api/images.go b/cmd/api/api/images.go index 4cc7cd93..1a367540 100644 --- a/cmd/api/api/images.go +++ b/cmd/api/api/images.go @@ -102,6 +102,7 @@ func (s *ApiService) DeleteImage(ctx context.Context, request oapi.DeleteImageRe func imageToOAPI(img images.Image) oapi.Image { oapiImg := oapi.Image{ Name: img.Name, + Digest: img.Digest, Status: oapi.ImageStatus(img.Status), QueuePosition: img.QueuePosition, Error: img.Error, diff --git a/lib/images/README.md b/lib/images/README.md index efbd468a..95dfb270 100644 --- a/lib/images/README.md +++ b/lib/images/README.md @@ -48,30 +48,74 @@ OCI Registry → containers/image → OCI Layout → umoci → rootfs/ → mkfs. **Alternative:** ext4 without journal works but erofs is optimized for this exact use case -## Filesystem Layout (storage.go) +## Filesystem Layout (storage.go, oci.go) +Content-addressable storage with tag symlinks (similar to Docker/Unikraft): ``` /var/lib/hypeman/ images/ docker.io/library/alpine/ - latest/ - metadata.json # Status, entrypoint, cmd, env - rootfs.erofs # Compressed read-only disk - 3.18/ # Different version - system/oci-cache/ - docker.io/library/alpine/latest/ - blobs/sha256/... # Shared layers, persistent + abc123def456.../ # Digest (sha256:abc123def456...) + metadata.json # Status, entrypoint, cmd, env + rootfs.erofs # Compressed read-only disk + def456abc123.../ # Another version (digest) + metadata.json + rootfs.erofs + latest -> abc123def456... # Tag symlink to digest + 3.18 -> def456abc123... # Another tag + system/ + oci-cache/ # Shared OCI layout for all images + index.json # Manifest index with digest-based tags + blobs/sha256/ + 2d35eb... # Layer blobs (shared across all images!) + 44cf07... # Another layer + 706db5... # Config blob for alpine + abc123def456... # Manifest for alpine:latest ``` **Benefits:** -- Natural hierarchy (versions grouped) -- Layer caching (alpine:latest and alpine:3.18 share base layers) +- Content-addressable: Digests are immutable, same content stored once +- Tag mutability: Tags (symlinks) can point to different digests over time +- Deduplication: Multiple tags can point to same digest +- Natural hierarchy: All versions of an image grouped under repository +- Easy inspection: Clear which digest belongs to which image +- Layer caching: All images share the same blob storage, layers deduplicated automatically + +**Design:** +- Images stored by manifest digest (content hash) +- Tags are filesystem symlinks pointing to digest directories +- Manifest always inspected upfront to discover digest (validates existence) +- Pulling same tag twice updates the symlink if digest changed +- OCI cache uses digest hex as layout tag for true content-addressable caching +- Shared blob storage enables automatic layer deduplication across all images +- Old digests remain until explicitly garbage collected +- Symlinks only created after successful build (status: ready) + +## Reference Handling (reference.go) + +Two types for type-safe image reference handling: + +**`NormalizedRef`** - Validated format (parsing only): +```go +normalized, err := ParseNormalizedRef("alpine") +// Normalizes to "docker.io/library/alpine:latest" +``` + +**`ResolvedRef`** - Normalized + manifest digest (network call): +```go +resolved, err := normalized.Resolve(ctx, ociClient) +// Now has digest from registry inspection -## Input Validation +resolved.Repository() // "docker.io/library/alpine" +resolved.Tag() // "latest" +resolved.Digest() // "sha256:abc123..." (always present) +``` -Uses `github.com/distribution/reference` to validate and normalize names: +Validation via `github.com/distribution/reference`: - `alpine` → `docker.io/library/alpine:latest` +- `alpine:3.18` → `docker.io/library/alpine:3.18` +- `alpine@sha256:abc123...` → digest validated against registry - Rejects invalid formats (returns 400) ## Build Tags diff --git a/lib/images/manager.go b/lib/images/manager.go index 8f8a2a2e..5430cce5 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -8,8 +8,6 @@ import ( "sort" "sync" "time" - - "github.com/distribution/reference" ) const ( @@ -29,7 +27,7 @@ type Manager interface { } type manager struct { - dataDir string + dataDir string ociClient *ociClient queue *BuildQueue createMu sync.Mutex @@ -43,7 +41,7 @@ func NewManager(dataDir string, maxConcurrentBuilds int) (Manager, error) { if err != nil { return nil, fmt.Errorf("create oci client: %w", err) } - + m := &manager{ dataDir: dataDir, ociClient: ociClient, @@ -54,9 +52,9 @@ func NewManager(dataDir string, maxConcurrentBuilds int) (Manager, error) { } func (m *manager) ListImages(ctx context.Context) ([]Image, error) { - metas, err := listMetadata(m.dataDir) + metas, err := listAllTags(m.dataDir) if err != nil { - return nil, fmt.Errorf("list metadata: %w", err) + return nil, fmt.Errorf("list tags: %w", err) } images := make([]Image, 0, len(metas)) @@ -68,39 +66,58 @@ func (m *manager) ListImages(ctx context.Context) ([]Image, error) { } func (m *manager) CreateImage(ctx context.Context, req CreateImageRequest) (*Image, error) { - normalizedName, err := normalizeImageName(req.Name) + // Parse and normalize + normalized, err := ParseNormalizedRef(req.Name) if err != nil { return nil, fmt.Errorf("%w: %s", ErrInvalidName, err.Error()) } + // Resolve to get digest (validates existence) + ref, err := normalized.Resolve(ctx, m.ociClient) + if err != nil { + return nil, fmt.Errorf("resolve manifest: %w", err) + } + m.createMu.Lock() defer m.createMu.Unlock() - // Check if already exists (handles ready, failed, and in-progress) - if meta, err := readMetadata(m.dataDir, normalizedName); err == nil { + // Check if we already have this digest (deduplication) + if meta, err := readMetadata(m.dataDir, ref.Repository(), ref.DigestHex()); err == nil { + // We have this digest already + if meta.Status == StatusReady && ref.Tag() != "" { + // Update tag symlink to point to current digest + // (handles case where tag moved to new digest) + createTagSymlink(m.dataDir, ref.Repository(), ref.Tag(), ref.DigestHex()) + } img := meta.toImage() - // Add dynamic queue position only for pending status + // Add queue position if pending if meta.Status == StatusPending { - img.QueuePosition = m.queue.GetPosition(normalizedName) + img.QueuePosition = m.queue.GetPosition(meta.Digest) } return img, nil } - // Create metadata (we know it doesn't exist) + // Don't have this digest yet, queue the build + return m.createAndQueueImage(ref) +} + +func (m *manager) createAndQueueImage(ref *ResolvedRef) (*Image, error) { meta := &imageMetadata{ - Name: normalizedName, + Name: ref.String(), + Digest: ref.Digest(), Status: StatusPending, - Request: &req, + Request: &CreateImageRequest{Name: ref.String()}, CreatedAt: time.Now(), } - if err := writeMetadata(m.dataDir, normalizedName, meta); err != nil { + // Write initial metadata + if err := writeMetadata(m.dataDir, ref.Repository(), ref.DigestHex(), meta); err != nil { return nil, fmt.Errorf("write initial metadata: %w", err) } - // Enqueue (we know it's not already queued due to mutex) - queuePos := m.queue.Enqueue(normalizedName, req, func() { - m.buildImage(context.Background(), normalizedName, req) + // Enqueue the build using digest as the queue key for deduplication + queuePos := m.queue.Enqueue(ref.Digest(), CreateImageRequest{Name: ref.String()}, func() { + m.buildImage(context.Background(), ref) }) img := meta.toImage() @@ -110,75 +127,107 @@ func (m *manager) CreateImage(ctx context.Context, req CreateImageRequest) (*Ima return img, nil } -func (m *manager) buildImage(ctx context.Context, imageName string, req CreateImageRequest) { - - buildDir := filepath.Join(imageDir(m.dataDir, imageName), ".build") +func (m *manager) buildImage(ctx context.Context, ref *ResolvedRef) { + buildDir := filepath.Join(m.dataDir, "system", "builds", ref.String()) tempDir := filepath.Join(buildDir, "rootfs") if err := os.MkdirAll(buildDir, 0755); err != nil { - m.updateStatus(imageName, StatusFailed, fmt.Errorf("create build dir: %w", err)) + m.updateStatusByDigest(ref, StatusFailed, fmt.Errorf("create build dir: %w", err)) return } defer func() { - meta, _ := readMetadata(m.dataDir, imageName) - if meta != nil && meta.Status == StatusReady { - os.RemoveAll(buildDir) - } + // Clean up build directory after completion + os.RemoveAll(buildDir) }() - m.updateStatus(imageName, StatusPulling, nil) - containerMeta, err := m.ociClient.pullAndExport(ctx, req.Name, tempDir) + m.updateStatusByDigest(ref, StatusPulling, nil) + + // Pull the image (digest is always known, uses cache if already pulled) + result, err := m.ociClient.pullAndExport(ctx, ref.String(), ref.Digest(), tempDir) if err != nil { - m.updateStatus(imageName, StatusFailed, fmt.Errorf("pull and export: %w", err)) + m.updateStatusByDigest(ref, StatusFailed, fmt.Errorf("pull and export: %w", err)) return } - m.updateStatus(imageName, StatusConverting, nil) - diskPath := imagePath(m.dataDir, imageName) + // Check if this digest already exists and is ready (deduplication) + if meta, err := readMetadata(m.dataDir, ref.Repository(), ref.DigestHex()); err == nil { + if meta.Status == StatusReady { + // Another build completed first, just update the tag symlink + if ref.Tag() != "" { + createTagSymlink(m.dataDir, ref.Repository(), ref.Tag(), ref.DigestHex()) + } + return + } + } + + m.updateStatusByDigest(ref, StatusConverting, nil) + + diskPath := digestPath(m.dataDir, ref.Repository(), ref.DigestHex()) diskSize, err := convertToErofs(tempDir, diskPath) if err != nil { - m.updateStatus(imageName, StatusFailed, fmt.Errorf("convert to erofs: %w", err)) + m.updateStatusByDigest(ref, StatusFailed, fmt.Errorf("convert to erofs: %w", err)) return } - meta, err := readMetadata(m.dataDir, imageName) + // Read current metadata to preserve request info + meta, err := readMetadata(m.dataDir, ref.Repository(), ref.DigestHex()) if err != nil { - m.updateStatus(imageName, StatusFailed, fmt.Errorf("read metadata: %w", err)) - return + // Create new metadata if it doesn't exist + meta = &imageMetadata{ + Name: ref.String(), + Digest: ref.Digest(), + CreatedAt: time.Now(), + } } + // Update with final status meta.Status = StatusReady meta.Error = nil meta.SizeBytes = diskSize - meta.Entrypoint = containerMeta.Entrypoint - meta.Cmd = containerMeta.Cmd - meta.Env = containerMeta.Env - meta.WorkingDir = containerMeta.WorkingDir + meta.Entrypoint = result.Metadata.Entrypoint + meta.Cmd = result.Metadata.Cmd + meta.Env = result.Metadata.Env + meta.WorkingDir = result.Metadata.WorkingDir - if err := writeMetadata(m.dataDir, imageName, meta); err != nil { - m.updateStatus(imageName, StatusFailed, fmt.Errorf("write final metadata: %w", err)) + if err := writeMetadata(m.dataDir, ref.Repository(), ref.DigestHex(), meta); err != nil { + m.updateStatusByDigest(ref, StatusFailed, fmt.Errorf("write final metadata: %w", err)) return } + + // Only create/update tag symlink on successful completion + if ref.Tag() != "" { + if err := createTagSymlink(m.dataDir, ref.Repository(), ref.Tag(), ref.DigestHex()); err != nil { + // Log error but don't fail the build + fmt.Fprintf(os.Stderr, "Warning: failed to create tag symlink: %v\n", err) + } + } } -func (m *manager) updateStatus(imageName, status string, err error) { - meta, readErr := readMetadata(m.dataDir, imageName) +func (m *manager) updateStatusByDigest(ref *ResolvedRef, status string, err error) { + meta, readErr := readMetadata(m.dataDir, ref.Repository(), ref.DigestHex()) if readErr != nil { - return + // Create new metadata if it doesn't exist + meta = &imageMetadata{ + Name: ref.String(), + Digest: ref.Digest(), + Status: status, + CreatedAt: time.Now(), + } + } else { + meta.Status = status } - meta.Status = status if err != nil { errorMsg := err.Error() meta.Error = &errorMsg } - writeMetadata(m.dataDir, imageName, meta) + writeMetadata(m.dataDir, ref.Repository(), ref.DigestHex(), meta) } func (m *manager) RecoverInterruptedBuilds() { - metas, err := listMetadata(m.dataDir) + metas, err := listAllTags(m.dataDir) if err != nil { return // Best effort } @@ -191,10 +240,16 @@ func (m *manager) RecoverInterruptedBuilds() { for _, meta := range metas { switch meta.Status { case StatusPending, StatusPulling, StatusConverting: - if meta.Request != nil { + if meta.Request != nil && meta.Digest != "" { metaCopy := meta - m.queue.Enqueue(metaCopy.Name, *metaCopy.Request, func() { - m.buildImage(context.Background(), metaCopy.Name, *metaCopy.Request) + normalized, err := ParseNormalizedRef(metaCopy.Name) + if err != nil { + continue + } + // Create a ResolvedRef since we already have the digest from metadata + ref := NewResolvedRef(normalized, metaCopy.Digest) + m.queue.Enqueue(metaCopy.Digest, *metaCopy.Request, func() { + m.buildImage(context.Background(), ref) }) } } @@ -202,45 +257,57 @@ func (m *manager) RecoverInterruptedBuilds() { } func (m *manager) GetImage(ctx context.Context, name string) (*Image, error) { - normalizedName, err := normalizeImageName(name) + // Parse and normalize the reference + ref, err := ParseNormalizedRef(name) if err != nil { return nil, fmt.Errorf("%w: %s", ErrInvalidName, err.Error()) } - meta, err := readMetadata(m.dataDir, normalizedName) + repository := ref.Repository() + + var digestHex string + + if ref.IsDigest() { + // Direct digest lookup + digestHex = ref.DigestHex() + } else { + // Tag lookup - resolve symlink + tag := ref.Tag() + + digestHex, err = resolveTag(m.dataDir, repository, tag) + if err != nil { + return nil, err + } + } + + meta, err := readMetadata(m.dataDir, repository, digestHex) if err != nil { return nil, err } - + img := meta.toImage() - + if meta.Status == StatusPending { - img.QueuePosition = m.queue.GetPosition(normalizedName) + img.QueuePosition = m.queue.GetPosition(meta.Digest) } - + return img, nil } func (m *manager) DeleteImage(ctx context.Context, name string) error { - normalizedName, err := normalizeImageName(name) + // Parse and normalize the reference + ref, err := ParseNormalizedRef(name) if err != nil { return fmt.Errorf("%w: %s", ErrInvalidName, err.Error()) } - return deleteImage(m.dataDir, normalizedName) -} -// normalizeImageName validates and normalizes an OCI image reference -// Examples: alpine → docker.io/library/alpine:latest -// nginx:1.0 → docker.io/library/nginx:1.0 -func normalizeImageName(name string) (string, error) { - named, err := reference.ParseNormalizedNamed(name) - if err != nil { - return "", err + // Only allow deleting by tag, not by digest + if ref.IsDigest() { + return fmt.Errorf("cannot delete by digest, use tag name instead") } - - // Ensure it has a tag (add :latest if missing) - tagged := reference.TagNameOnly(named) - return tagged.String(), nil -} + repository := ref.Repository() + tag := ref.Tag() + return deleteTag(m.dataDir, repository, tag) +} diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go index eaccc5c5..a9d31944 100644 --- a/lib/images/manager_test.go +++ b/lib/images/manager_test.go @@ -3,7 +3,7 @@ package images import ( "context" "os" - "path/filepath" + "strings" "testing" "time" @@ -32,10 +32,21 @@ func TestCreateImage(t *testing.T) { require.Equal(t, StatusReady, img.Status) require.NotNil(t, img.SizeBytes) require.Greater(t, *img.SizeBytes, int64(0)) + require.NotEmpty(t, img.Digest) + require.Contains(t, img.Digest, "sha256:") - diskPath := filepath.Join(dataDir, "images", imageNameToPath(img.Name), "rootfs.erofs") + // Check that digest directory exists + ref, err := ParseNormalizedRef(img.Name) + require.NoError(t, err) + digestHex := strings.SplitN(img.Digest, ":", 2)[1] + diskPath := digestPath(dataDir, ref.Repository(), digestHex) _, err = os.Stat(diskPath) require.NoError(t, err) + + // Check that tag symlink exists + linkPath := tagSymlinkPath(dataDir, ref.Repository(), ref.Tag()) + _, err = os.Lstat(linkPath) + require.NoError(t, err) } func TestCreateImageDifferentTag(t *testing.T) { @@ -54,6 +65,10 @@ func TestCreateImageDifferentTag(t *testing.T) { require.Equal(t, "docker.io/library/alpine:3.18", img.Name) waitForReady(t, mgr, ctx, img.Name) + + img, err = mgr.GetImage(ctx, img.Name) + require.NoError(t, err) + require.NotEmpty(t, img.Digest) } func TestCreateImageDuplicate(t *testing.T) { @@ -71,12 +86,18 @@ func TestCreateImageDuplicate(t *testing.T) { waitForReady(t, mgr, ctx, img1.Name) + // Re-fetch img1 to get the complete metadata including digest + img1, err = mgr.GetImage(ctx, img1.Name) + require.NoError(t, err) + require.NotEmpty(t, img1.Digest) + // Second create should be idempotent and return existing image img2, err := mgr.CreateImage(ctx, req) require.NoError(t, err) require.NotNil(t, img2) require.Equal(t, img1.Name, img2.Name) require.Equal(t, StatusReady, img2.Status) + require.Equal(t, img1.Digest, img2.Digest) // Same digest } func TestListImages(t *testing.T) { @@ -105,6 +126,7 @@ func TestListImages(t *testing.T) { require.Len(t, images, 1) require.Equal(t, "docker.io/library/alpine:latest", images[0].Name) require.Equal(t, StatusReady, images[0].Status) + require.NotEmpty(t, images[0].Digest) } func TestGetImage(t *testing.T) { @@ -128,6 +150,7 @@ func TestGetImage(t *testing.T) { require.Equal(t, created.Name, img.Name) require.Equal(t, StatusReady, img.Status) require.NotNil(t, img.SizeBytes) + require.NotEmpty(t, img.Digest) } func TestGetImageNotFound(t *testing.T) { @@ -156,15 +179,24 @@ func TestDeleteImage(t *testing.T) { waitForReady(t, mgr, ctx, created.Name) + // Get the digest before deleting + img, err := mgr.GetImage(ctx, created.Name) + require.NoError(t, err) + ref, err := ParseNormalizedRef(img.Name) + require.NoError(t, err) + digestHex := strings.SplitN(img.Digest, ":", 2)[1] + err = mgr.DeleteImage(ctx, created.Name) require.NoError(t, err) + // Tag should be gone _, err = mgr.GetImage(ctx, created.Name) require.ErrorIs(t, err, ErrNotFound) - imageDir := filepath.Join(dataDir, "images", imageNameToPath(created.Name)) - _, err = os.Stat(imageDir) - require.True(t, os.IsNotExist(err)) + // But digest directory should still exist + digestDir := digestPath(dataDir, ref.Repository(), digestHex) + _, err = os.Stat(digestDir) + require.NoError(t, err) } func TestDeleteImageNotFound(t *testing.T) { @@ -178,22 +210,28 @@ func TestDeleteImageNotFound(t *testing.T) { require.ErrorIs(t, err, ErrNotFound) } -func TestImageNameToPath(t *testing.T) { +func TestNormalizedRefParsing(t *testing.T) { tests := []struct { - input string - expected string + input string + expectRepo string + expectTag string }{ - {"docker.io/library/nginx:latest", "docker.io/library/nginx/latest"}, - {"docker.io/library/alpine:3.18", "docker.io/library/alpine/3.18"}, - {"gcr.io/my-project/my-app:v1.0.0", "gcr.io/my-project/my-app/v1.0.0"}, - {"nginx", "nginx/latest"}, - {"ubuntu:22.04", "ubuntu/22.04"}, + {"alpine", "docker.io/library/alpine", "latest"}, + {"alpine:3.18", "docker.io/library/alpine", "3.18"}, + {"docker.io/library/alpine:latest", "docker.io/library/alpine", "latest"}, + {"ghcr.io/myorg/myapp:v1.0.0", "ghcr.io/myorg/myapp", "v1.0.0"}, } for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { - result := imageNameToPath(tt.input) - require.Equal(t, tt.expected, result) + ref, err := ParseNormalizedRef(tt.input) + require.NoError(t, err) + + repo := ref.Repository() + require.Equal(t, tt.expectRepo, repo) + + tag := ref.Tag() + require.Equal(t, tt.expectTag, tag) }) } } @@ -228,5 +266,3 @@ func waitForReady(t *testing.T, mgr Manager, ctx context.Context, imageName stri t.Fatal("Build did not complete within 60 seconds") } - - diff --git a/lib/images/oci.go b/lib/images/oci.go index bdf7e6b4..ec22e38c 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -4,10 +4,11 @@ import ( "context" "fmt" "os" - "path/filepath" + "strings" "github.com/containers/image/v5/copy" "github.com/containers/image/v5/docker" + "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/oci/layout" "github.com/containers/image/v5/signature" "github.com/opencontainers/image-spec/specs-go/v1" @@ -22,6 +23,35 @@ type ociClient struct { cacheDir string } +// digestToLayoutTag converts a digest to a valid OCI layout tag. +// Uses just the hex portion without the algorithm prefix. +// Example: "sha256:abc123..." -> "abc123..." +func digestToLayoutTag(digest string) string { + // Extract just the hex hash after the colon + parts := strings.SplitN(digest, ":", 2) + if len(parts) == 2 { + return parts[1] + } + return digest // Fallback if no colon found +} + +// existsInLayout checks if a digest already exists in the OCI layout cache. +func (c *ociClient) existsInLayout(layoutTag string) bool { + casEngine, err := dir.Open(c.cacheDir) + if err != nil { + return false + } + defer casEngine.Close() + + engine := casext.NewEngine(casEngine) + descriptorPaths, err := engine.ResolveReference(context.Background(), layoutTag) + if err != nil { + return false + } + + return len(descriptorPaths) > 0 +} + // newOCIClient creates a new OCI client func newOCIClient(cacheDir string) (*ociClient, error) { if err := os.MkdirAll(cacheDir, 0755); err != nil { @@ -30,38 +60,88 @@ func newOCIClient(cacheDir string) (*ociClient, error) { return &ociClient{cacheDir: cacheDir}, nil } -func (c *ociClient) pullAndExport(ctx context.Context, imageRef, exportDir string) (*containerMetadata, error) { - // Use persistent OCI layout for caching (parse imageRef into path) - ociLayoutDir := filepath.Join(c.cacheDir, imageNameToPath(imageRef)) - if err := os.MkdirAll(ociLayoutDir, 0755); err != nil { - return nil, fmt.Errorf("create oci layout dir: %w", err) +// inspectManifest synchronously inspects a remote image to get its digest +// without pulling the image. This is used for upfront digest discovery. +func (c *ociClient) inspectManifest(ctx context.Context, imageRef string) (string, error) { + srcRef, err := docker.ParseReference("//" + imageRef) + if err != nil { + return "", fmt.Errorf("parse image reference: %w", err) } - if err := c.pullToOCILayout(ctx, imageRef, ociLayoutDir); err != nil { - return nil, fmt.Errorf("pull to oci layout: %w", err) + // Create image source to inspect the remote manifest + src, err := srcRef.NewImageSource(ctx, nil) + if err != nil { + return "", fmt.Errorf("create image source: %w", err) } + defer src.Close() - meta, err := c.extractOCIMetadata(ociLayoutDir) + // Get the manifest bytes + manifestBytes, manifestType, err := src.GetManifest(ctx, nil) + if err != nil { + return "", fmt.Errorf("get manifest: %w", err) + } + + // Compute digest of the manifest + // For multi-arch images, this returns the manifest list digest + manifestDigest, err := manifest.Digest(manifestBytes) + if err != nil { + return "", fmt.Errorf("compute manifest digest: %w", err) + } + + // Note: manifestType tells us if this is a manifest list or single-platform manifest + _ = manifestType + + return manifestDigest.String(), nil +} + +// pullResult contains the metadata and digest from pulling an image +type pullResult struct { + Metadata *containerMetadata + Digest string // sha256:abc123... +} + +func (c *ociClient) pullAndExport(ctx context.Context, imageRef, digest, exportDir string) (*pullResult, error) { + // Use a shared OCI layout for all images to enable automatic layer caching + // The cacheDir itself is the OCI layout root with shared blobs/sha256/ directory + // The digest is ALWAYS known at this point (from inspectManifest or digest reference) + layoutTag := digestToLayoutTag(digest) + + // Check if this digest is already cached + if !c.existsInLayout(layoutTag) { + // Not cached, pull it using digest-based tag + if err := c.pullToOCILayout(ctx, imageRef, layoutTag); err != nil { + return nil, fmt.Errorf("pull to oci layout: %w", err) + } + } + // If cached, we skip the pull entirely + + // Extract metadata (from cache or freshly pulled) + meta, err := c.extractOCIMetadata(layoutTag) if err != nil { return nil, fmt.Errorf("extract metadata: %w", err) } - if err := c.unpackLayers(ctx, ociLayoutDir, exportDir); err != nil { + // Unpack layers to the export directory + if err := c.unpackLayers(ctx, layoutTag, exportDir); err != nil { return nil, fmt.Errorf("unpack layers: %w", err) } - return meta, nil + return &pullResult{ + Metadata: meta, + Digest: digest, + }, nil } -func (c *ociClient) pullToOCILayout(ctx context.Context, imageRef, ociLayoutDir string) error { +func (c *ociClient) pullToOCILayout(ctx context.Context, imageRef, layoutTag string) error { // Parse source reference (docker://...) srcRef, err := docker.ParseReference("//" + imageRef) if err != nil { return fmt.Errorf("parse image reference: %w", err) } - // Create destination reference (OCI layout) - destRef, err := layout.ParseReference(ociLayoutDir + ":latest") + // Create destination reference (shared OCI layout with sanitized tag) + // This allows multiple images to coexist in the same layout with automatic layer deduplication + destRef, err := layout.ParseReference(c.cacheDir + ":" + layoutTag) if err != nil { return fmt.Errorf("parse oci layout reference: %w", err) } @@ -85,10 +165,35 @@ func (c *ociClient) pullToOCILayout(ctx context.Context, imageRef, ociLayoutDir return nil } +// extractDigest gets the manifest digest from the OCI layout +func (c *ociClient) extractDigest(layoutTag string) (string, error) { + casEngine, err := dir.Open(c.cacheDir) + if err != nil { + return "", fmt.Errorf("open oci layout: %w", err) + } + defer casEngine.Close() + + engine := casext.NewEngine(casEngine) + + // Resolve the layout tag in the shared layout + descriptorPaths, err := engine.ResolveReference(context.Background(), layoutTag) + if err != nil { + return "", fmt.Errorf("resolve reference: %w", err) + } + + if len(descriptorPaths) == 0 { + return "", fmt.Errorf("no image found in oci layout") + } + + // Get the manifest descriptor's digest + digest := descriptorPaths[0].Descriptor().Digest.String() + return digest, nil +} + // extractOCIMetadata reads metadata from OCI layout config.json -func (c *ociClient) extractOCIMetadata(ociLayoutDir string) (*containerMetadata, error) { - // Open the OCI layout - casEngine, err := dir.Open(ociLayoutDir) +func (c *ociClient) extractOCIMetadata(layoutTag string) (*containerMetadata, error) { + // Open the shared OCI layout + casEngine, err := dir.Open(c.cacheDir) if err != nil { return nil, fmt.Errorf("open oci layout: %w", err) } @@ -96,8 +201,8 @@ func (c *ociClient) extractOCIMetadata(ociLayoutDir string) (*containerMetadata, engine := casext.NewEngine(casEngine) - // Get image reference (we tagged it as "latest") - descriptorPaths, err := engine.ResolveReference(context.Background(), "latest") + // Resolve the layout tag in the shared layout + descriptorPaths, err := engine.ResolveReference(context.Background(), layoutTag) if err != nil { return nil, fmt.Errorf("resolve reference: %w", err) } @@ -154,9 +259,9 @@ func (c *ociClient) extractOCIMetadata(ociLayoutDir string) (*containerMetadata, } // unpackLayers unpacks all OCI layers to a target directory using umoci -func (c *ociClient) unpackLayers(ctx context.Context, ociLayoutDir, targetDir string) error { - // Open OCI layout - casEngine, err := dir.Open(ociLayoutDir) +func (c *ociClient) unpackLayers(ctx context.Context, imageRef, targetDir string) error { + // Open the shared OCI layout + casEngine, err := dir.Open(c.cacheDir) if err != nil { return fmt.Errorf("open oci layout: %w", err) } @@ -164,8 +269,8 @@ func (c *ociClient) unpackLayers(ctx context.Context, ociLayoutDir, targetDir st engine := casext.NewEngine(casEngine) - // Get the manifest descriptor for "latest" tag - descriptorPaths, err := engine.ResolveReference(context.Background(), "latest") + // Resolve the image reference (tag) in the shared layout + descriptorPaths, err := engine.ResolveReference(context.Background(), imageRef) if err != nil { return fmt.Errorf("resolve reference: %w", err) } diff --git a/lib/images/reference.go b/lib/images/reference.go new file mode 100644 index 00000000..8dd23ff2 --- /dev/null +++ b/lib/images/reference.go @@ -0,0 +1,160 @@ +package images + +import ( + "context" + "strings" + + "github.com/distribution/reference" +) + +// NormalizedRef is a validated and normalized OCI image reference. +// It can be either a tagged reference (e.g., "docker.io/library/alpine:latest") +// or a digest reference (e.g., "docker.io/library/alpine@sha256:abc123..."). +type NormalizedRef struct { + raw string + repository string + tag string // empty if digest ref + digest string // empty if tag ref + isDigest bool +} + +// ParseNormalizedRef validates and normalizes a user-provided image reference. +// Examples: +// - "alpine" -> "docker.io/library/alpine:latest" +// - "alpine:3.18" -> "docker.io/library/alpine:3.18" +// - "alpine@sha256:abc..." -> "docker.io/library/alpine@sha256:abc..." +func ParseNormalizedRef(s string) (*NormalizedRef, error) { + named, err := reference.ParseNormalizedNamed(s) + if err != nil { + return nil, err + } + + ref := &NormalizedRef{} + + // Extract repository (always present) + ref.repository = reference.Domain(named) + "/" + reference.Path(named) + + // If it's canonical (has digest), extract digest + if canonical, ok := named.(reference.Canonical); ok { + ref.isDigest = true + ref.digest = canonical.Digest().String() + ref.raw = canonical.String() + return ref, nil + } + + // Otherwise it's a tagged reference - ensure tag (add :latest if missing) + tagged := reference.TagNameOnly(named) + if t, ok := tagged.(reference.Tagged); ok { + ref.tag = t.Tag() + } + ref.raw = tagged.String() + + return ref, nil +} + +// String returns the full normalized reference. +func (r *NormalizedRef) String() string { + return r.raw +} + +// IsDigest returns true if this reference contains a digest (@sha256:...). +func (r *NormalizedRef) IsDigest() bool { + return r.isDigest +} + +// Digest returns the digest if present (e.g., "sha256:abc123..."). +// Returns empty string if this is a tagged reference. +func (r *NormalizedRef) Digest() string { + return r.digest +} + +// Repository returns the repository path without tag or digest. +// Example: "docker.io/library/alpine" +func (r *NormalizedRef) Repository() string { + return r.repository +} + +// Tag returns the tag if this is a tagged reference (e.g., "latest"). +// Returns empty string if this is a digest reference. +func (r *NormalizedRef) Tag() string { + return r.tag +} + +// DigestHex returns just the hex portion of the digest (without "sha256:" prefix). +// Returns empty string if this is a tagged reference. +func (r *NormalizedRef) DigestHex() string { + if r.digest == "" { + return "" + } + + // Strip "sha256:" prefix + parts := strings.SplitN(r.digest, ":", 2) + if len(parts) != 2 { + return "" // Invalid format + } + + return parts[1] +} + +// ResolvedRef is a NormalizedRef that has been resolved to include the actual +// manifest digest from the registry. The digest is always present. +type ResolvedRef struct { + normalized *NormalizedRef + digest string // Always populated (e.g., "sha256:abc123...") +} + +// NewResolvedRef creates a ResolvedRef from a NormalizedRef and digest. +func NewResolvedRef(normalized *NormalizedRef, digest string) *ResolvedRef { + return &ResolvedRef{ + normalized: normalized, + digest: digest, + } +} + +// String returns the full normalized reference (the original user input format). +func (r *ResolvedRef) String() string { + return r.normalized.String() +} + +// Repository returns the repository path without tag or digest. +// Example: "docker.io/library/alpine" +func (r *ResolvedRef) Repository() string { + return r.normalized.Repository() +} + +// Tag returns the tag if this was originally a tagged reference (e.g., "latest"). +// Returns empty string if this was originally a digest reference. +func (r *ResolvedRef) Tag() string { + return r.normalized.Tag() +} + +// Digest returns the resolved manifest digest (e.g., "sha256:abc123..."). +// This is always populated after resolution. +func (r *ResolvedRef) Digest() string { + return r.digest +} + +// DigestHex returns just the hex portion of the digest (without "sha256:" prefix). +func (r *ResolvedRef) DigestHex() string { + // Strip "sha256:" prefix + parts := strings.SplitN(r.digest, ":", 2) + if len(parts) != 2 { + return "" // Invalid format + } + return parts[1] +} + +// Resolve inspects the manifest to get the digest and returns a ResolvedRef. +// This requires an ociClient interface for manifest inspection. +type ManifestInspector interface { + inspectManifest(ctx context.Context, imageRef string) (string, error) +} + +// Resolve returns a ResolvedRef by inspecting the manifest to get the authoritative digest. +func (r *NormalizedRef) Resolve(ctx context.Context, inspector ManifestInspector) (*ResolvedRef, error) { + digest, err := inspector.inspectManifest(ctx, r.String()) + if err != nil { + return nil, err + } + return NewResolvedRef(r, digest), nil +} diff --git a/lib/images/storage.go b/lib/images/storage.go index a3d14dd2..10ee921c 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -10,21 +10,23 @@ import ( ) type imageMetadata struct { - Name string `json:"name"` - Status string `json:"status"` - Error *string `json:"error,omitempty"` - Request *CreateImageRequest `json:"request,omitempty"` - SizeBytes int64 `json:"size_bytes"` - Entrypoint []string `json:"entrypoint,omitempty"` - Cmd []string `json:"cmd,omitempty"` - Env map[string]string `json:"env,omitempty"` - WorkingDir string `json:"working_dir,omitempty"` - CreatedAt time.Time `json:"created_at"` + Name string `json:"name"` // Normalized ref (tag or digest) + Digest string `json:"digest"` // Always present: sha256:... + Status string `json:"status"` + Error *string `json:"error,omitempty"` + Request *CreateImageRequest `json:"request,omitempty"` + SizeBytes int64 `json:"size_bytes"` + Entrypoint []string `json:"entrypoint,omitempty"` + Cmd []string `json:"cmd,omitempty"` + Env map[string]string `json:"env,omitempty"` + WorkingDir string `json:"working_dir,omitempty"` + CreatedAt time.Time `json:"created_at"` } func (m *imageMetadata) toImage() *Image { img := &Image{ Name: m.Name, + Digest: m.Digest, Status: m.Status, Error: m.Error, CreatedAt: m.CreatedAt, @@ -51,37 +53,33 @@ func (m *imageMetadata) toImage() *Image { return img } -// imageNameToPath converts image name to nested directory structure -// docker.io/library/alpine:latest → docker.io/library/alpine/latest -func imageNameToPath(name string) string { - // Split on last colon to separate tag - lastColon := strings.LastIndex(name, ":") - if lastColon == -1 { - // No tag, use "latest" - return filepath.Join(name, "latest") - } - - imagePath := name[:lastColon] - tag := name[lastColon+1:] - return filepath.Join(imagePath, tag) +// digestDir returns the directory for a specific digest +// e.g., /var/lib/hypeman/images/docker.io/library/alpine/abc123def456... +func digestDir(dataDir, repository, digestHex string) string { + return filepath.Join(dataDir, "images", repository, digestHex) } -func imageDir(dataDir, imageName string) string { - return filepath.Join(dataDir, "images", imageNameToPath(imageName)) +// digestPath returns the path to the rootfs.erofs file for a digest +func digestPath(dataDir, repository, digestHex string) string { + return filepath.Join(digestDir(dataDir, repository, digestHex), "rootfs.erofs") } -func imagePath(dataDir, imageName string) string { - return filepath.Join(imageDir(dataDir, imageName), "rootfs.erofs") +// metadataPath returns the path to metadata.json for a digest +func metadataPath(dataDir, repository, digestHex string) string { + return filepath.Join(digestDir(dataDir, repository, digestHex), "metadata.json") } -func metadataPath(dataDir, imageName string) string { - return filepath.Join(imageDir(dataDir, imageName), "metadata.json") +// tagSymlinkPath returns the path to a tag symlink +// e.g., /var/lib/hypeman/images/docker.io/library/alpine/latest +func tagSymlinkPath(dataDir, repository, tag string) string { + return filepath.Join(dataDir, "images", repository, tag) } -func writeMetadata(dataDir, imageName string, meta *imageMetadata) error { - dir := imageDir(dataDir, imageName) +// writeMetadata writes metadata for a digest +func writeMetadata(dataDir, repository, digestHex string, meta *imageMetadata) error { + dir := digestDir(dataDir, repository, digestHex) if err := os.MkdirAll(dir, 0755); err != nil { - return fmt.Errorf("create image directory: %w", err) + return fmt.Errorf("create digest directory: %w", err) } data, err := json.MarshalIndent(meta, "", " ") @@ -89,12 +87,12 @@ func writeMetadata(dataDir, imageName string, meta *imageMetadata) error { return fmt.Errorf("marshal metadata: %w", err) } - tempPath := metadataPath(dataDir, imageName) + ".tmp" + tempPath := metadataPath(dataDir, repository, digestHex) + ".tmp" if err := os.WriteFile(tempPath, data, 0644); err != nil { return fmt.Errorf("write temp metadata: %w", err) } - finalPath := metadataPath(dataDir, imageName) + finalPath := metadataPath(dataDir, repository, digestHex) if err := os.Rename(tempPath, finalPath); err != nil { os.Remove(tempPath) return fmt.Errorf("rename metadata: %w", err) @@ -103,8 +101,9 @@ func writeMetadata(dataDir, imageName string, meta *imageMetadata) error { return nil } -func readMetadata(dataDir, imageName string) (*imageMetadata, error) { - path := metadataPath(dataDir, imageName) +// readMetadata reads metadata for a digest +func readMetadata(dataDir, repository, digestHex string) (*imageMetadata, error) { + path := metadataPath(dataDir, repository, digestHex) data, err := os.ReadFile(path) if err != nil { if os.IsNotExist(err) { @@ -119,7 +118,7 @@ func readMetadata(dataDir, imageName string) (*imageMetadata, error) { } if meta.Status == StatusReady { - diskPath := imagePath(dataDir, imageName) + diskPath := digestPath(dataDir, repository, digestHex) if _, err := os.Stat(diskPath); err != nil { if os.IsNotExist(err) { return nil, fmt.Errorf("disk image missing: %s", diskPath) @@ -131,30 +130,111 @@ func readMetadata(dataDir, imageName string) (*imageMetadata, error) { return &meta, nil } -func listMetadata(dataDir string) ([]*imageMetadata, error) { +// createTagSymlink creates or updates a tag symlink to point to a digest +// Only creates the symlink if the digest dir exists and build is ready +func createTagSymlink(dataDir, repository, tag, digestHex string) error { + linkPath := tagSymlinkPath(dataDir, repository, tag) + targetPath := digestHex // Relative path (just the digest hex) + + // Ensure parent directory exists + if err := os.MkdirAll(filepath.Dir(linkPath), 0755); err != nil { + return fmt.Errorf("create parent directory: %w", err) + } + + // Remove existing symlink if present + os.Remove(linkPath) + + // Create new symlink + if err := os.Symlink(targetPath, linkPath); err != nil { + return fmt.Errorf("create symlink: %w", err) + } + + return nil +} + +// resolveTag follows a tag symlink to get the digest hex +func resolveTag(dataDir, repository, tag string) (string, error) { + linkPath := tagSymlinkPath(dataDir, repository, tag) + + // Read the symlink + target, err := os.Readlink(linkPath) + if err != nil { + if os.IsNotExist(err) { + return "", ErrNotFound + } + return "", fmt.Errorf("read symlink: %w", err) + } + + // Validate it's just a digest hex (not an absolute path) + if filepath.IsAbs(target) || strings.Contains(target, "/") { + return "", fmt.Errorf("invalid symlink target: %s", target) + } + + return target, nil +} + +// listTags returns all tags for a repository +func listTags(dataDir, repository string) ([]string, error) { + repoDir := filepath.Join(dataDir, "images", repository) + + entries, err := os.ReadDir(repoDir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read repository directory: %w", err) + } + + var tags []string + for _, entry := range entries { + // Check if it's a symlink + info, err := os.Lstat(filepath.Join(repoDir, entry.Name())) + if err != nil { + continue + } + + if info.Mode()&os.ModeSymlink != 0 { + tags = append(tags, entry.Name()) + } + } + + return tags, nil +} + +// listAllTags returns all tags across all repositories +func listAllTags(dataDir string) ([]*imageMetadata, error) { imagesDir := filepath.Join(dataDir, "images") var metas []*imageMetadata + // Walk the images directory to find all repositories err := filepath.Walk(imagesDir, func(path string, info os.FileInfo, err error) error { if err != nil { return nil // Skip errors } - - if info.Name() == "metadata.json" { - // Read metadata file - data, err := os.ReadFile(path) + + // Check if this is a symlink (tag) + if info.Mode()&os.ModeSymlink != 0 { + // Read the symlink to get digest hex + digestHex, err := os.Readlink(path) if err != nil { - return nil + return nil // Skip invalid symlinks } - - var meta imageMetadata - if err := json.Unmarshal(data, &meta); err != nil { + + // Get repository from path + relPath, err := filepath.Rel(imagesDir, filepath.Dir(path)) + if err != nil { return nil } - - metas = append(metas, &meta) + + // Read metadata for this digest + meta, err := readMetadata(dataDir, relPath, digestHex) + if err != nil { + return nil // Skip if metadata can't be read + } + + metas = append(metas, meta) } - + return nil }) @@ -165,22 +245,28 @@ func listMetadata(dataDir string) ([]*imageMetadata, error) { return metas, nil } -func imageExists(dataDir, imageName string) bool { - _, err := readMetadata(dataDir, imageName) +// digestExists checks if a digest directory exists +func digestExists(dataDir, repository, digestHex string) bool { + dir := digestDir(dataDir, repository, digestHex) + _, err := os.Stat(dir) return err == nil } -func deleteImage(dataDir, imageName string) error { - dir := imageDir(dataDir, imageName) - if _, err := os.Stat(dir); err != nil { +// deleteTag removes a tag symlink (does not delete the digest directory) +func deleteTag(dataDir, repository, tag string) error { + linkPath := tagSymlinkPath(dataDir, repository, tag) + + // Check if symlink exists + if _, err := os.Lstat(linkPath); err != nil { if os.IsNotExist(err) { return ErrNotFound } - return fmt.Errorf("stat image directory: %w", err) + return fmt.Errorf("stat symlink: %w", err) } - if err := os.RemoveAll(dir); err != nil { - return fmt.Errorf("remove image directory: %w", err) + // Remove symlink + if err := os.Remove(linkPath); err != nil { + return fmt.Errorf("remove symlink: %w", err) } return nil diff --git a/lib/images/types.go b/lib/images/types.go index 6e5fcd55..6b8a99a7 100644 --- a/lib/images/types.go +++ b/lib/images/types.go @@ -4,7 +4,8 @@ import "time" // Image represents a container image converted to bootable disk type Image struct { - Name string + Name string // Normalized ref (e.g., docker.io/library/alpine:latest) + Digest string // Resolved manifest digest (sha256:...) Status string QueuePosition *int Error *string diff --git a/lib/images/validation_test.go b/lib/images/validation_test.go index 9b37e074..891c3cf9 100644 --- a/lib/images/validation_test.go +++ b/lib/images/validation_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestNormalizeImageName(t *testing.T) { +func TestParseNormalizedRef(t *testing.T) { tests := []struct { input string expected string @@ -15,17 +15,21 @@ func TestNormalizeImageName(t *testing.T) { // Valid images with full reference {"docker.io/library/alpine:latest", "docker.io/library/alpine:latest", false}, {"ghcr.io/myorg/myapp:v1.0.0", "ghcr.io/myorg/myapp:v1.0.0", false}, - + // Shorthand (gets expanded) {"alpine", "docker.io/library/alpine:latest", false}, {"alpine:3.18", "docker.io/library/alpine:3.18", false}, {"nginx", "docker.io/library/nginx:latest", false}, {"nginx:alpine", "docker.io/library/nginx:alpine", false}, - + // Without tag (gets :latest added) {"docker.io/library/alpine", "docker.io/library/alpine:latest", false}, {"ubuntu", "docker.io/library/ubuntu:latest", false}, - + + // Digest references (must be valid 64-char hex SHA256) + {"alpine@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "docker.io/library/alpine@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", false}, + {"docker.io/library/alpine@sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210", "docker.io/library/alpine@sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210", false}, + // Invalid {"", "", true}, {"invalid::", "", true}, @@ -35,14 +39,61 @@ func TestNormalizeImageName(t *testing.T) { for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { - result, err := normalizeImageName(tt.input) + result, err := ParseNormalizedRef(tt.input) if tt.wantErr { require.Error(t, err) } else { require.NoError(t, err) - require.Equal(t, tt.expected, result) + require.Equal(t, tt.expected, result.String()) } }) } } +func TestNormalizedRefMethods(t *testing.T) { + t.Run("TaggedReference", func(t *testing.T) { + ref, err := ParseNormalizedRef("alpine:3.18") + require.NoError(t, err) + + require.False(t, ref.IsDigest()) + + repo := ref.Repository() + require.Equal(t, "docker.io/library/alpine", repo) + + tag := ref.Tag() + require.Equal(t, "3.18", tag) + + digest := ref.Digest() + require.Equal(t, "", digest) + + digestHex := ref.DigestHex() + require.Equal(t, "", digestHex) + }) + + t.Run("DigestReference", func(t *testing.T) { + ref, err := ParseNormalizedRef("alpine@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef") + require.NoError(t, err) + + require.True(t, ref.IsDigest()) + + repo := ref.Repository() + require.Equal(t, "docker.io/library/alpine", repo) + + tag := ref.Tag() + require.Equal(t, "", tag) + + digest := ref.Digest() + require.Equal(t, "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", digest) + + digestHex := ref.DigestHex() + require.Equal(t, "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", digestHex) + }) + + t.Run("DefaultTag", func(t *testing.T) { + ref, err := ParseNormalizedRef("alpine") + require.NoError(t, err) + + tag := ref.Tag() + require.Equal(t, "latest", tag) + }) +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 75847b34..98ac09a5 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -164,7 +164,10 @@ type Image struct { // Error Error message if status is failed Error *string `json:"error"` - // Name OCI image reference (also serves as unique identifier) + // Digest Resolved manifest digest + Digest string `json:"digest"` + + // Name Normalized OCI image reference (tag or digest) Name string `json:"name"` // QueuePosition Position in build queue (null if not queued) @@ -176,9 +179,6 @@ type Image struct { // Status Build status Status ImageStatus `json:"status"` - // Version Image tag or digest - Version *string `json:"version"` - // WorkingDir Working directory from container metadata WorkingDir *string `json:"working_dir"` } diff --git a/openapi.yaml b/openapi.yaml index fd444d21..aed503c7 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -236,12 +236,16 @@ components: Image: type: object - required: [name, status, created_at] + required: [name, digest, status, created_at] properties: name: type: string - description: OCI image reference (also serves as unique identifier) + description: Normalized OCI image reference (tag or digest) example: docker.io/library/nginx:latest + digest: + type: string + description: Resolved manifest digest + example: sha256:abc123def456... status: type: string enum: [pending, pulling, converting, ready, failed] @@ -257,11 +261,6 @@ components: description: Error message if status is failed example: "pull failed: connection timeout" nullable: true - version: - type: string - description: Image tag or digest - example: latest - nullable: true size_bytes: type: integer format: int64 From 37fac0b4cd01bcb9c583c1bb47102a7428b62696 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 15:15:14 -0500 Subject: [PATCH 27/37] WIP: figuring out error handling registry errors --- lib/images/manager.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/images/manager.go b/lib/images/manager.go index 5430cce5..bcc3c2aa 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -73,7 +73,10 @@ func (m *manager) CreateImage(ctx context.Context, req CreateImageRequest) (*Ima } // Resolve to get digest (validates existence) - ref, err := normalized.Resolve(ctx, m.ociClient) + resolveCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + ref, err := normalized.Resolve(resolveCtx, m.ociClient) if err != nil { return nil, fmt.Errorf("resolve manifest: %w", err) } From d8bab8a45d2e58cb9b97005ef0509c877a0429e6 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 15:25:58 -0500 Subject: [PATCH 28/37] Switch to more lightweight library for interacting with registry --- go.mod | 50 ++-------- go.sum | 206 ++---------------------------------------- lib/images/manager.go | 5 +- lib/images/oci.go | 73 ++++++--------- 4 files changed, 48 insertions(+), 286 deletions(-) diff --git a/go.mod b/go.mod index c964ca4a..87a6cb87 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,12 @@ module github.com/onkernel/hypeman go 1.25.4 require ( - github.com/containers/image/v5 v5.36.2 + github.com/distribution/reference v0.6.0 github.com/getkin/kin-openapi v0.133.0 github.com/ghodss/yaml v1.0.0 github.com/go-chi/chi/v5 v5.2.3 github.com/golang-jwt/jwt/v5 v5.3.0 + github.com/google/go-containerregistry v0.20.6 github.com/google/wire v0.7.0 github.com/joho/godotenv v1.5.1 github.com/oapi-codegen/nethttp-middleware v1.1.2 @@ -21,46 +22,27 @@ require ( require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect - github.com/BurntSushi/toml v1.5.0 // indirect - github.com/VividCortex/ewma v1.2.0 // indirect - github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/apex/log v1.9.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect - github.com/containers/ocicrypt v1.2.1 // indirect - github.com/containers/storage v1.59.1 // indirect - github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect github.com/cyphar/filepath-securejoin v0.5.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/distribution/reference v0.6.0 // indirect + github.com/docker/cli v28.2.2+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/docker v28.5.2+incompatible // indirect github.com/docker/docker-credential-helpers v0.9.3 // indirect - github.com/docker/go-connections v0.5.0 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/go-jose/go-jose/v4 v4.0.5 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/go-containerregistry v0.20.3 // indirect + github.com/go-test/deep v1.1.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/pgzip v1.2.6 // indirect - github.com/letsencrypt/boulder v0.0.0-20240620165639-de9c06129bec // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-runewidth v0.0.16 // indirect - github.com/mattn/go-sqlite3 v1.14.28 // indirect - github.com/miekg/pkcs11 v1.1.1 // indirect - github.com/moby/sys/capability v0.4.0 // indirect - github.com/moby/sys/mountinfo v0.7.2 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect @@ -68,34 +50,16 @@ require ( github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/proglottis/gpgme v0.1.4 // indirect - github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/rootless-containers/proto/go-proto v0.0.0-20230421021042-4cd87ebadd67 // indirect - github.com/secure-systems-lab/go-securesystemslib v0.9.0 // indirect - github.com/sigstore/fulcio v1.6.6 // indirect - github.com/sigstore/protobuf-specs v0.4.1 // indirect - github.com/sigstore/sigstore v1.9.5 // indirect github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af // indirect - github.com/smallstep/pkcs7 v0.1.1 // indirect - github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect - github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect - github.com/ulikunitz/xz v0.5.12 // indirect github.com/vbatts/go-mtree v0.6.1-0.20250911112631-8307d76bc1b9 // indirect github.com/vbatts/tar-split v0.12.1 // indirect - github.com/vbauerster/mpb/v8 v8.10.2 // indirect github.com/woodsbury/decimal128 v1.3.0 // indirect - go.opentelemetry.io/otel v1.38.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect golang.org/x/crypto v0.41.0 // indirect - golang.org/x/net v0.42.0 // indirect golang.org/x/sys v0.37.0 // indirect - golang.org/x/term v0.34.0 // indirect - golang.org/x/text v0.28.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250313205543-e70fdf4c4cb4 // indirect - google.golang.org/grpc v1.72.2 // indirect google.golang.org/protobuf v1.36.10 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gotest.tools/v3 v3.5.2 // indirect ) diff --git a/go.sum b/go.sum index 39ecf526..fde2cb2e 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,6 @@ -dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= -dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= -github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= -github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= -github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= @@ -18,27 +10,11 @@ github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= -github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containers/image/v5 v5.36.2 h1:GcxYQyAHRF/pLqR4p4RpvKllnNL8mOBn0eZnqJbfTwk= -github.com/containers/image/v5 v5.36.2/go.mod h1:b4GMKH2z/5t6/09utbse2ZiLK/c72GuGLFdp7K69eA4= -github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA= -github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= -github.com/containers/ocicrypt v1.2.1 h1:0qIOTT9DoYwcKmxSt8QJt+VzMY18onl9jUXsxpVhSmM= -github.com/containers/ocicrypt v1.2.1/go.mod h1:aD0AAqfMp0MtwqWgHM1bUwe1anx0VazI108CRrSKINQ= -github.com/containers/storage v1.59.1 h1:11Zu68MXsEQGBBd+GadPrHPpWeqjKS8hJDGiAHgIqDs= -github.com/containers/storage v1.59.1/go.mod h1:KoAYHnAjP3/cTsRS+mmWZGkufSY2GACiKQ4V3ZLQnR0= -github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 h1:uX1JmpONuD549D73r6cgnxyUu18Zb7yHAy5AYU0Pm4Q= -github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw= +github.com/containerd/stargz-snapshotter/estargz v0.16.3 h1:7evrXtoh1mSbGj/pfRccTampEyKpjpOnS3CyiV1Ebr8= +github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9WBVE8cOlQmXAbPN9VEQpBBeJIuOipU= github.com/cyphar/filepath-securejoin v0.5.0 h1:hIAhkRBMQ8nIeuVwcAoymp7MY4oherZdAxD+m0u9zaw= github.com/cyphar/filepath-securejoin v0.5.0/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -47,23 +23,13 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/cli v28.3.2+incompatible h1:mOt9fcLE7zaACbxW1GeS65RI67wIJrTnqS3hP2huFsY= -github.com/docker/cli v28.3.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v28.2.2+incompatible h1:qzx5BNUDFqlvyq4AHzdNB7gSyVTmU4cgsyN9SdInc1A= +github.com/docker/cli v28.2.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= -github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= -github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= -github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/getkin/kin-openapi v0.133.0 h1:pJdmNohVIJ97r4AUFtEXRXwESr8b0bD721u/Tz6k8PQ= github.com/getkin/kin-openapi v0.133.0/go.mod h1:boAciF6cXk5FhPqe/NQeBTeenbjqU4LhWBf09ILVvWE= @@ -71,13 +37,7 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE= github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= -github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= -github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= @@ -89,17 +49,11 @@ github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArs github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-containerregistry v0.20.3 h1:oNx7IdTI936V8CQRveCjaxOiegWwvM7kqkbXTpyiovI= -github.com/google/go-containerregistry v0.20.3/go.mod h1:w00pIgBRDVUDFM6bq+Qx8lwNWK+cxgCuX1vd3PIBDNI= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= -github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/go-containerregistry v0.20.6 h1:cvWX87UxxLgaH76b4hIvya6Dzz9qHB31qAwjAohdSTU= +github.com/google/go-containerregistry v0.20.6/go.mod h1:T0x8MuoAoKX/873bkeSfLD2FAkwCDf9/HZgsFJ02E2Y= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -109,15 +63,11 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmhodges/clock v1.2.0 h1:eq4kys+NI0PLngzaHEe7AmPT90XMGIEySD1JfV1PDIs= -github.com/jmhodges/clock v1.2.0/go.mod h1:qKjhA7x7u/lQpPB1XAqX1b1lCI/w3/fNuYpI/ZjLynI= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= @@ -131,38 +81,21 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/letsencrypt/boulder v0.0.0-20240620165639-de9c06129bec h1:2tTW6cDth2TSgRbAhD7yjZzTQmcN25sDRPEeinR51yQ= -github.com/letsencrypt/boulder v0.0.0-20240620165639-de9c06129bec/go.mod h1:TmwEoGCwIti7BCeJ9hescZgRtatxRE+A72pCoPfmcfk= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= -github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= -github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= -github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= -github.com/moby/sys/capability v0.4.0 h1:4D4mI6KlNtWMCM1Z/K0i7RV1FkX+DBDHKVJpCndZoHk= -github.com/moby/sys/capability v0.4.0/go.mod h1:4g9IK291rVkms3LKCDOoYlnV8xKwoDTpIrNEE35Wq0I= -github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= -github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/oapi-codegen/nethttp-middleware v1.1.2 h1:TQwEU3WM6ifc7ObBEtiJgbRPaCe513tvJpiMJjypVPA= github.com/oapi-codegen/nethttp-middleware v1.1.2/go.mod h1:5qzjxMSiI8HjLljiOEjvs4RdrWyMPKnExeFS2kr8om4= github.com/oapi-codegen/runtime v1.1.2 h1:P2+CubHq8fO4Q6fV1tqDBZHCwpVpvPg7oKiYzQgXIyI= @@ -189,53 +122,24 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/proglottis/gpgme v0.1.4 h1:3nE7YNA70o2aLjcg63tXMOhPD7bplfE5CBdV+hLAm2M= -github.com/proglottis/gpgme v0.1.4/go.mod h1:5LoXMgpE4bttgwwdv9bLs/vwqv3qV7F4glEEZ7mRKrM= -github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= -github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= -github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= -github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= -github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rootless-containers/proto/go-proto v0.0.0-20230421021042-4cd87ebadd67 h1:58jvc5cZ+hGKidQ4Z37/+rj9eQxRRjOOsqNEwPSZXR4= github.com/rootless-containers/proto/go-proto v0.0.0-20230421021042-4cd87ebadd67/go.mod h1:LLjEAc6zmycfeN7/1fxIphWQPjHpTt7ElqT7eVf8e4A= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= -github.com/secure-systems-lab/go-securesystemslib v0.9.0 h1:rf1HIbL64nUpEIZnjLZ3mcNEL9NBPB0iuVjyxvq3LZc= -github.com/secure-systems-lab/go-securesystemslib v0.9.0/go.mod h1:DVHKMcZ+V4/woA/peqr+L0joiRXbPpQ042GgJckkFgw= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sigstore/fulcio v1.6.6 h1:XaMYX6TNT+8n7Npe8D94nyZ7/ERjEsNGFC+REdi/wzw= -github.com/sigstore/fulcio v1.6.6/go.mod h1:BhQ22lwaebDgIxVBEYOOqLRcN5+xOV+C9bh/GUXRhOk= -github.com/sigstore/protobuf-specs v0.4.1 h1:5SsMqZbdkcO/DNHudaxuCUEjj6x29tS2Xby1BxGU7Zc= -github.com/sigstore/protobuf-specs v0.4.1/go.mod h1:+gXR+38nIa2oEupqDdzg4qSBT0Os+sP7oYv6alWewWc= -github.com/sigstore/sigstore v1.9.5 h1:Wm1LT9yF4LhQdEMy5A2JeGRHTrAWGjT3ubE5JUSrGVU= -github.com/sigstore/sigstore v1.9.5/go.mod h1:VtxgvGqCmEZN9X2zhFSOkfXxvKUjpy8RpUW39oCtoII= github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af h1:Sp5TG9f7K39yfB+If0vjp97vuT74F72r8hfRpP8jLU0= github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smallstep/pkcs7 v0.1.1 h1:x+rPdt2W088V9Vkjho4KtoggyktZJlMduZAtRHm68LU= -github.com/smallstep/pkcs7 v0.1.1/go.mod h1:dL6j5AIz9GHjVEBTXtW+QliALcgM19RtXaTeyxI+AfA= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= -github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 h1:pnnLyeX7o/5aX8qUQ69P/mLojDqwda8hFOCBTmP/6hw= -github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6/go.mod h1:39R/xuhNgVhi+K0/zst4TLrJrVmbm6LVgl4A0+ZFS5M= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C1wj2THlRK+oAhjeS/TRQwMfkIuet3w0= -github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= @@ -245,125 +149,33 @@ github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPf github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= -github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/vbatts/go-mtree v0.6.1-0.20250911112631-8307d76bc1b9 h1:R6l9BtUe83abUGu1YKGkfa17wMMFLt6mhHVQ8MxpfRE= github.com/vbatts/go-mtree v0.6.1-0.20250911112631-8307d76bc1b9/go.mod h1:W7bcG9PCn6lFY+ljGlZxx9DONkxL3v8a7HyN+PrSrjA= github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo= github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= -github.com/vbauerster/mpb/v8 v8.10.2 h1:2uBykSHAYHekE11YvJhKxYmLATKHAGorZwFlyNw4hHM= -github.com/vbauerster/mpb/v8 v8.10.2/go.mod h1:+Ja4P92E3/CorSZgfDtK46D7AVbDqmBQRTmyTqPElo0= github.com/woodsbury/decimal128 v1.3.0 h1:8pffMNWIlC0O5vbyHWFZAt5yWvWcrHA+3ovIIjVWss0= github.com/woodsbury/decimal128 v1.3.0/go.mod h1:C5UTmyTjW3JftjUFzOVhC20BEQa2a4ZKOB5I6Zjb+ds= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= -go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= -go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= -go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= -go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= -go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= -go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= -go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= -go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= -go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= -golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= -golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250313205543-e70fdf4c4cb4 h1:iK2jbkWL86DXjEx0qiHcRE9dE4/Ahua5k6V8OWFb//c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250313205543-e70fdf4c4cb4/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= -google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= -google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= @@ -382,3 +194,5 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/lib/images/manager.go b/lib/images/manager.go index bcc3c2aa..5430cce5 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -73,10 +73,7 @@ func (m *manager) CreateImage(ctx context.Context, req CreateImageRequest) (*Ima } // Resolve to get digest (validates existence) - resolveCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - - ref, err := normalized.Resolve(resolveCtx, m.ociClient) + ref, err := normalized.Resolve(ctx, m.ociClient) if err != nil { return nil, fmt.Errorf("resolve manifest: %w", err) } diff --git a/lib/images/oci.go b/lib/images/oci.go index ec22e38c..003e87fd 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -6,11 +6,10 @@ import ( "os" "strings" - "github.com/containers/image/v5/copy" - "github.com/containers/image/v5/docker" - "github.com/containers/image/v5/manifest" - "github.com/containers/image/v5/oci/layout" - "github.com/containers/image/v5/signature" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/opencontainers/image-spec/specs-go/v1" rspec "github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/umoci/oci/cas/dir" @@ -63,35 +62,19 @@ func newOCIClient(cacheDir string) (*ociClient, error) { // inspectManifest synchronously inspects a remote image to get its digest // without pulling the image. This is used for upfront digest discovery. func (c *ociClient) inspectManifest(ctx context.Context, imageRef string) (string, error) { - srcRef, err := docker.ParseReference("//" + imageRef) + ref, err := name.ParseReference(imageRef) if err != nil { return "", fmt.Errorf("parse image reference: %w", err) } - // Create image source to inspect the remote manifest - src, err := srcRef.NewImageSource(ctx, nil) + // Head request to get manifest descriptor - no automatic retries + // Rate limits return immediately with actual error + descriptor, err := remote.Head(ref, remote.WithContext(ctx)) if err != nil { - return "", fmt.Errorf("create image source: %w", err) + return "", fmt.Errorf("fetch manifest: %w", err) } - defer src.Close() - // Get the manifest bytes - manifestBytes, manifestType, err := src.GetManifest(ctx, nil) - if err != nil { - return "", fmt.Errorf("get manifest: %w", err) - } - - // Compute digest of the manifest - // For multi-arch images, this returns the manifest list digest - manifestDigest, err := manifest.Digest(manifestBytes) - if err != nil { - return "", fmt.Errorf("compute manifest digest: %w", err) - } - - // Note: manifestType tells us if this is a manifest list or single-platform manifest - _ = manifestType - - return manifestDigest.String(), nil + return descriptor.Digest.String(), nil } // pullResult contains the metadata and digest from pulling an image @@ -133,33 +116,37 @@ func (c *ociClient) pullAndExport(ctx context.Context, imageRef, digest, exportD } func (c *ociClient) pullToOCILayout(ctx context.Context, imageRef, layoutTag string) error { - // Parse source reference (docker://...) - srcRef, err := docker.ParseReference("//" + imageRef) + ref, err := name.ParseReference(imageRef) if err != nil { return fmt.Errorf("parse image reference: %w", err) } - // Create destination reference (shared OCI layout with sanitized tag) - // This allows multiple images to coexist in the same layout with automatic layer deduplication - destRef, err := layout.ParseReference(c.cacheDir + ":" + layoutTag) + // Fetch image manifest from registry (lazy - doesn't download layers yet) + img, err := remote.Image(ref, remote.WithContext(ctx)) if err != nil { - return fmt.Errorf("parse oci layout reference: %w", err) + // Rate limits fail here immediately during manifest fetch + return fmt.Errorf("fetch image manifest: %w", err) } - // Create policy context (allow all) - policyContext, err := signature.NewPolicyContext(&signature.Policy{ - Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}, - }) + // Open or create OCI layout directory + path, err := layout.FromPath(c.cacheDir) if err != nil { - return fmt.Errorf("create policy context: %w", err) + // If layout doesn't exist, create it + path, err = layout.Write(c.cacheDir, empty.Index) + if err != nil { + return fmt.Errorf("create oci layout: %w", err) + } } - defer policyContext.Destroy() - _, err = copy.Image(ctx, policyContext, destRef, srcRef, ©.Options{ - ReportWriter: os.Stdout, - }) + // Append image to layout - THIS is where actual layer data is downloaded + // Streams layers from registry and writes to blobs/sha256/ directory + // Automatically deduplicates shared layers across images + // Rate limits during layer download also fail immediately (no retries) + err = path.AppendImage(img, layout.WithAnnotations(map[string]string{ + "org.opencontainers.image.ref.name": layoutTag, + })) if err != nil { - return fmt.Errorf("copy image: %w", err) + return fmt.Errorf("download and write image layers: %w", err) } return nil From 0c0f38ab865c62c2c783d42049add97db4512ebe Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 15:34:29 -0500 Subject: [PATCH 29/37] Handle 404 --- cmd/api/api/images.go | 7 ++ cmd/api/api/images_test.go | 43 +++--------- lib/oapi/oapi.go | 132 +++++++++++++++++++++---------------- openapi.yaml | 6 ++ 4 files changed, 96 insertions(+), 92 deletions(-) diff --git a/cmd/api/api/images.go b/cmd/api/api/images.go index 1a367540..740d5746 100644 --- a/cmd/api/api/images.go +++ b/cmd/api/api/images.go @@ -3,6 +3,8 @@ package api import ( "context" "errors" + "fmt" + "strings" "github.com/onkernel/hypeman/lib/images" "github.com/onkernel/hypeman/lib/logger" @@ -44,6 +46,11 @@ func (s *ApiService) CreateImage(ctx context.Context, request oapi.CreateImageRe Code: "invalid_name", Message: err.Error(), }, nil + case strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "not found"): + return oapi.CreateImage404JSONResponse{ + Code: "not_found", + Message: fmt.Sprintf("image not found: %v", err), + }, nil default: log.Error("failed to create image", "error", err, "name", request.Body.Name) return oapi.CreateImage500JSONResponse{ diff --git a/cmd/api/api/images_test.go b/cmd/api/api/images_test.go index 2a181fe6..686cfbb9 100644 --- a/cmd/api/api/images_test.go +++ b/cmd/api/api/images_test.go @@ -132,41 +132,14 @@ func TestCreateImage_InvalidTag(t *testing.T) { }) require.NoError(t, err) - acceptedResp, ok := createResp.(oapi.CreateImage202JSONResponse) - require.True(t, ok, "expected 202 accepted response") - - img := oapi.Image(acceptedResp) - require.Equal(t, "docker.io/library/busybox:foobar", img.Name) - t.Logf("Image created: name=%s", img.Name) - - // Poll until failed - t.Log("Polling for failure...") - for i := 0; i < 1000; i++ { - getResp, err := svc.GetImage(ctx, oapi.GetImageRequestObject{Name: img.Name}) - require.NoError(t, err) - - imgResp, ok := getResp.(oapi.GetImage200JSONResponse) - require.True(t, ok, "expected 200 response") - - currentImg := oapi.Image(imgResp) - t.Logf("Status: %s", currentImg.Status) - - if currentImg.Status == oapi.ImageStatus(images.StatusFailed) { - t.Log("Build failed as expected") - require.NotNil(t, currentImg.Error) - require.Contains(t, *currentImg.Error, "foobar") - t.Logf("Error message: %s", *currentImg.Error) - return - } - - if currentImg.Status == oapi.ImageStatus(images.StatusReady) { - t.Fatal("Build should have failed but succeeded") - } - - time.Sleep(10 * time.Millisecond) - } - - t.Fatal("Build did not fail within timeout") + // With go-containerregistry, manifest validation happens synchronously + // Invalid tags fail immediately with 404 (manifest not found) + errorResp, ok := createResp.(oapi.CreateImage404JSONResponse) + require.True(t, ok, "expected 404 not found response for invalid tag") + + errObj := oapi.Error(errorResp) + require.Equal(t, "not_found", errObj.Code) + t.Logf("Got expected error: code=%s message=%s", errObj.Code, errObj.Message) } func TestCreateImage_InvalidName(t *testing.T) { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 98ac09a5..46ef4ad6 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -155,6 +155,9 @@ type Image struct { // CreatedAt Creation timestamp (RFC3339) CreatedAt time.Time `json:"created_at"` + // Digest Resolved manifest digest + Digest string `json:"digest"` + // Entrypoint Entrypoint from container metadata Entrypoint *[]string `json:"entrypoint"` @@ -164,9 +167,6 @@ type Image struct { // Error Error message if status is failed Error *string `json:"error"` - // Digest Resolved manifest digest - Digest string `json:"digest"` - // Name Normalized OCI image reference (tag or digest) Name string `json:"name"` @@ -1547,6 +1547,7 @@ type CreateImageResponse struct { JSON202 *Image JSON400 *Error JSON401 *Error + JSON404 *Error JSON500 *Error } @@ -2223,6 +2224,13 @@ func ParseCreateImageResponse(rsp *http.Response) (*CreateImageResponse, error) } response.JSON401 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: var dest Error if err := json.Unmarshal(bodyBytes, &dest); err != nil { @@ -3796,6 +3804,15 @@ func (response CreateImage401JSONResponse) VisitCreateImageResponse(w http.Respo return json.NewEncoder(w).Encode(response) } +type CreateImage404JSONResponse Error + +func (response CreateImage404JSONResponse) VisitCreateImageResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + + return json.NewEncoder(w).Encode(response) +} + type CreateImage500JSONResponse Error func (response CreateImage500JSONResponse) VisitCreateImageResponse(w http.ResponseWriter) error { @@ -4978,60 +4995,61 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xc+28Tu/L/Vyx/v0cqUtI8WjiQ+1MpcKhEoaLQI10Oipz1JPHBa29tbyBU/d+v/NjN", - "btZ5FNpcem4lJLJZe8Yz/szDM06vcCLTTAoQRuPBFdbJFFLiPh4ZQ5LpheR5Cu/hMgdt7NeZkhkow8AN", - "SmUuzDAjZmqfKOhEscwwKfAAnxEzRV+noADNHBWkpzLnFI0AuXlAcQvDN5JmHPAAd1JhOpQYglvYzDP7", - "lTaKiQm+bmEFhErB557NmOTc4MGYcA2tJbanljQiGtkpbTenpDeSkgMR+NpRvMyZAooHn6pifC4Hy9Hf", - "kBjL/FgBMXCSkslqTQiSQlMH745PELPzkIIxKBAJoD3Yn+y3EJXJF1D7THY4Gymi5h0xYeLbgBMD2jyq", - "qWb92Ka+lsRza1sjmNCGiGS1bCBm9j9CKbNyEX5We93YrLoOXooZU1KkIAyaEcXIiIOuineF37578XL4", - "8u0FHljONE/c1BY+e/f+Ax7gg263a+k21s9oU+UfBbvMATEKwrAxA4XGUiEzBcSCnGgvU3LGKFA0mqOE", - "cA6qrm87sk1GSa9/EAOj29EmZweQCuM6yXTSTqZKptCe9WJEU0ilmg9T8m2YjmowP+w+e9JAOfnG0jxF", - "fhb6yswUTaXJeD5BTKDT51XmnkDgyISBCagqyzq7Xrd/uMzuOdFQ8GqQ73cPn8bIx03idZ4S0bbGaYGA", - "3KCqotJ5+6tUX7gktB1VVCaVGaYky5iY6Ijbkcqg4jUaK5miqdQGGYkmubcWZiB1M/9fwRgP8P91Fl6w", - "E1xgx9I59WQq2CNKkbl7ZinI3Aw1JFJQXdPgwZNud1mDH/x4B0adEA5tI9vfQUmkISXCsKRmE7/3LYmm", - "TmdJlteZ9Zc5vc3TESgkx2jGlMkJR8dnH2vE+1HKzkdHFOpDgLYKJC4mbKtBP9HHEWv9TTUu+SlmA0IA", - "hLex1U5rQ2CKeYZ3mfdeKMm1kWnVReyR3Mj2BAQoYoAiNkZCGlT4ibp3mEnetnEqDs846v1yo2h3pLz2", - "Y/Q0+w7DyahJ8px9tz4NTdiEjOam7lN7EfTEosKCfkzVL5WSqqncRNKIiEdZxllC7FNbZ5CwMUsQWArI", - "TkB7KUmmTEBp+3WtjggdqrCdrVhMMYTxCDyPyqgUmIWRaM+aWppzwzIO/p1+tC12neQvHKWY9TMhQA2h", - "UM8NKKWgdTR6LPnFQpZyiPMcFEb5ZGJVUlXdKdOaiQkqdheNGXA68JnHxuzA7eZiYStxEGTYEg1v5FdQ", - "bQ4z4FUQeIuyi02lAlTixG/aUhCeEc7okIksN/GIuUKVr3JlplAgAZGRdbw2A/AbVmXiY7a19bHMBY0q", - "q6GO10C4z3jrmtCGmDxkTHlqdSu/WH0u2MkvG7cjEIltw0mRdyxtQBpxdsenL3zwS6QwhAlQKAVDQn5d", - "rugTdpkkbuG2xRQlkEqB5Hj8L7uC0lSaXi7n3OIUD4zKoWkgiXPSdEhMZGn2nUW0jaHakDRDe+9fHR8c", - "HDyru4R+t/+43e21e48/9LqDrv33b9zCY6lSSxdTYqBticTQAcKoeSaZiKzgZfluOx11fALeXtDc19Of", - "U9Ad5NTbyHKFz44+vLanrVyrDpcJ4R09YmJQeS4fFy/cB/84YiKai5fOcGmlzvaDqdq46vGNmEZjwvjS", - "GTDLOQ/fD6wkApISKdJ5gRV63RR/owcxwrVNv9QMtD0w5ssnh587hLXwZQ45DDOpmV9FM1X1b2wUH+WM", - "U+RmoD0rZJGDuK/qGUh/pRoq+ZyL6z4vaDB+wfQXpEP+4MYEnrkwjLuT87zG8fHBk6e/d5/1+hXrY8I8", - "OcRbLaX0i0unCidzeNsqnWYGgvoQZ+HgPyVSzKx1uAe3PusIPIBqHrZ419iMGSgd3QUfAgyZIBtg2cRv", - "5oJiub8boWfPLUxMhpRFDOFP/xJRpiAx9iC1hbniDsmyzaxXZHaFYivOOBpWwrk4Ellu34sf3NSL30Xp", - "oaGC8SUVsTyC8zm6zAm37oAiKlPCRDOJr5QL9p0L3QYsU6KHWpBMT2VEu39OwaUwBBVjEHxj2uhQzWC6", - "LGdUlxIqYsvlri0rJb9iDaQGOSnGbJLbDDet1z9+tOSxgvronpY7bqm2kSk2IwaGLIvw8+/QyRkilCrQ", - "tWMn7j3r7/eePN3vdbv7ve42dqANUat8zLl99wMO5vFKB7PNcgxs0l/hMc/dYDdLZtlKIWR2Ixn6G5zk", - "RhmitalYMSoJkCeh3nqT8tNdlpx8zQgoKkbsruJUIGDrqHleAGbJERbFZkdu8JdoI1+4ogN0cXqKAnU0", - "yo1L84IZoL1jLnOKXs8zUDOmpUKCGDaDR5bC+1wIJiaWgk2gSWLf8DlS/vv1k89Irj13OzdzT+tnnE9z", - "Q+VX4eboaW6QfXJLtiKEgLSehDeMAXor3Zyw0hYScjmy+eFE0NG8OXw5Cu4lRKCRzea1kQroo79EJYMM", - "msYtHDSGW9iLj1u4kMp+9Ktznxzjyk4vzKnqLRspkqspD62TXuGbmXClBzcOXZxWjeJp1Mamcj1B6Qna", - "YXVicXKZkkYmkteKxtgkWUVf/imnWUT+JYtZrK5VlT1mId4amyojwbqHRq6xm5MX9gRUjF2Tmmx0h7ed", - "xXaf3bQWcfPsa32NeV3b1fc/7buV+qt2Wn/wPP0L1rOrvrxgstGLNyLGT7a4mS562zXDv4tGd3FEaHJe", - "1/kuou4whsmwq2swuepAsLQXCx6t9c11CwhIcsXM/NwGca/zERAF6ij3OnfR3Qnhvl4wnxqT4etrV48f", - "R3zJHyBAsQQdnZ24Y1NKBJnYOHlxijgbQzJPOKDc1c4bQcy1VN8dn7TtYYCiIkl3x0dmnELs6JQISx9X", - "Cgy4u9/bdw1rmYEgGcMDfOC+amGrBidiZ1oWkSfgYGdB53zRCXVrN6HMbDWrMym0102/2/VVd2ECXsmi", - "8dL5W/sSh8+INuVLgYNT4ZIxWjUkDlV+oT530nmaEjW3srtvUTKF5It71XH5k14p0BumzYkf8pMSbZUK", - "+lp5M/9rSGrXZTPXsPzrFj7s9m5Nw76DFmH7UZDcTKVi34Fapo9vcVtXMj0RBpQg3Bc9VeiHVI0QDz7V", - "ze/T5+vP1X136lroKpM6steVCzPYOwbQ5rmk81sTMXIl57ruhGw4u24grX9rKwgAiyjZlUBGRW3TZ/VE", - "z0XyyKNrBxv9nFBUNFMfEL0e0Wc554gIikKNGZW9gqpf61zZ1OLaBxkO/shXR/0L932B+owokoIBpd0K", - "lnT1/k0bRCKpzRN8CzIcQu1bFyuLzKtIaerIblUUtxyKPzdQf7iqUOdFoR4jhzvYrqV26z2Cid/dAhit", - "lVH7J/bfXw1c3Az8rf8qtJp+678iPGMCfjs4WlwQvBuwdHflIosrHQ/g2wi+PyAE3YXSnGsKh+UNWVc5", - "aieJV9GWuEnuVa7wIVhtk35V1bU2A1u0iO4wCVu6PrxVHnZ7W7zAW0zhoZgUqgAP+dcvCGmPIpeBuXR5", - "0dis+7jOFaPb5F8LzC+F4Ei4dAWK286sCtDtPLkqGN/LEOfaZBYENCRalTiyMtfa6V53d+uzdp4e3Wv4", - "uAypobqmA+lwOdHrqm6FGt5I10+/dVy1GrdMJOfyK7LrQnvaKCCpLz6en78ss/zLHNR8wXPs5uAqn+Vi", - "cfMnT6u7tpwJf8FfgcmV8DeTwF2HjXEPV3UjvHuxtvEWpmTgm+nADIRpew3UQRW5k2snZJwwsX5kM+WU", - "ExRYPBjWdn7ZIbK0LY9Th82YeYWGrOujRDPT937AP9p1F13p/zLEDrvP7p71sRRjzhKD2guM2FUwYdM5", - "QUdzJFW13X+fwB/AupDMecYgVxT/xbuV+A83Df7R+F/s/f+4BSRSKUiMvwR0v4rilXSqYsp77t7Q4j5O", - "q0jXL05P4wEhXOHqXPkPJ5vOcItfnd9R9hUhUiztXlhZaNJTCDc7dm5hsrxzcE8L+VZxhQjOoVfPmnGv", - "Xf1rCPcBl7df7Iv9PYitSn07tYryvtOvYhW7jkBhDYS7H8LU9HFfDNQjrZDEyKWCYOXW8MqWx0V5b/ju", - "Gx7BKdyg3VFI8FAZ3qLZUVHWulZH6ZrvrtHxA77v9ja3QNlKz/fQ4vjlWxyzYg8XXmzLpsbdJR5btTTK", - "lHO3DY2LXyeeMn0vQ2m4tDIrQ9SqqvcuAdbdnVPcdQ/l4h6fi/6AIthW+ieOgKUYu8X0RiaEIwoz4DJz", - "P7L1Y3EL54qHC9qDjv+zAlOpjfuRCr7+fP2fAAAA//9FztPZeE4AAA==", + "H4sIAAAAAAAC/+xc/W8TOfP/Vyx/vycVKWleWjjI81MpcFSiULVcT3o4FDnrSeLDa29tbyBU/d8f+WU3", + "u1nnpdDm6B0SEtmsPeMZz8vHM06vcSLTTAoQRuPBNdbJFFLiPh4ZQ5LppeR5CudwlYM29utMyQyUYeAG", + "pTIXZpgRM7VPFHSiWGaYFHiAz4iZos9TUIBmjgrSU5lzikaA3DyguIXhC0kzDniAO6kwHUoMwS1s5pn9", + "ShvFxATftLACQqXgc89mTHJu8GBMuIbWEttTSxoRjeyUtptT0htJyYEIfOMoXuVMAcWDD1UxPpaD5egv", + "SIxlfqyAGDhJyWS1JgRJoamDd8cniNl5SMEYFIgE0B7sT/ZbiMrkE6h9JjucjRRR846YMPFlwIkBbR7V", + "VLN+bFNfS+K5ta0RTGhDRLJaNhAz+x+hlFm5CD+rvW5sVl0HL8WMKSlSEAbNiGJkxEFXxbvGb9+9eDl8", + "+fYSDyxnmiduagufvTt/jwf4oNvtWrqN9TPaVPnvgl3lgBgFYdiYgUJjqZCZAmJBTrSXKTljFCgazVFC", + "OAdV17cd2SajpNc/iBmj29EmZ2cgFcZ1kumknUyVTKE968WIppBKNR+m5MswHdXM/LD77EnDyskXluYp", + "8rPQZ2amaCpNxvMJYgKdPq8y9wQCRyYMTEBVWdbZ9br9w2V2z4mGgleDfL97+DRGPu4Sr/OUiLZ1TmsI", + "yA2qKiqdtz9L9YlLQttRRWVSmWFKsoyJiY6EHakMKl6jsZIpmkptkJFokntvYQZSN/P/FYzxAP9fZxEF", + "OyEEdiydU0+mYntEKTJ3zywFmZuhhkQKqmsaPHjS7S5r8L0f74xRJ4RD28j2V1ASaUiJMCyp+cSvfUui", + "qdNZkuV1Zv1lTm/zdAQKyTGaMWVywtHx2e814v0oZRejIwr1KUBbBRKXE7bVoJ/o84j1/qYal+IUswkh", + "GIT3sdVBa0NiikWGd5mPXijJtZFpNUTskdzI9gQEKGKAIjZGQhpUxIl6dJhJ3rZ5Km6ecav3y41auyPl", + "tR+jp9lXGE5GTZIX7KuNaWjCJmQ0N/WY2otYTywrLOjHVP1SKamayk0kjYh4lGWcJcQ+tXUGCRuzBIGl", + "gOwEtJeSZMoElL5f1+qI0KEK29mK5RRDGI+Y51GZlQKzMBLtWVdLc25YxsG/04+2tV0n+QtHKeb9TAhQ", + "QyjUcwtKKWgdzR5LcbGQpRziIgeFUT6ZWJVUVXfKtGZigordRWMGnA488tiIDtxuLha20g6CDFtawxv5", + "GVSbwwx41Qi8R9nFplIBKu3Eb9pSEp4RzuiQiSw38Yy5QpWvcmWmUFgCIiMbeC0C8BtWZeJztvX1scwF", + "jSqroY7XQLhHvHVNaENMHhBTnlrdyk9Wnwt28tPG7QhEYttwUuCOpQ1II8Hu+PSFT36JFIYwAQqlYEjA", + "1+WKPmCHJHELt61NUQKpFEiOx/+xKyhdpRnlcs6tneKBUTk0HSRxQZoOiYkszb6zFm1zqDYkzdDe+avj", + "g4ODZ/WQ0O/2H7e7vXbv8fted9C1//6LW3gsVWrpYkoMtC2RaMBgk5AZ6tzPQUs+A4pSItgYtEFhZJWz", + "npL+4ycDDwEpjA8fP9nf34+xAWHUPJNMRFi9LN9ttxUdj/PbC5r7evp9+3AP0H0bWa7x2dH71/ZQl2vV", + "4TIhvKNHTAwqz+Xj4oX74B9HTEQhfxlzl1bqQkyICDZ9ezdCTKMxYXzpqJnlnIfvB1YSAUlpkNIFmxV6", + "3ZTm31rT5OwrUBQ9+hkyQTaMO4v7vjNeC1/lkMMwk5p57k0k7N9YkDDKGafIzUB7VrgC4riv6gCnv1L8", + "Clx0sMHDjgbjF0x/QjrAEzcm8MyFYdwdzOc1jo8Pnjz9tfus1684NxPmySHeaill2F06tDiZw9tWGZMz", + "ENRnUGsG/lMixcx6hXtw67NxxhtOLYAX7xqbYc8sTEyGlEWs8w//ElGmIDH2ELWFD+EOybLNphhHdWVM", + "K8WvRORobgmH40h6uftQfnC7UH4/9YeGCsZXVMTABOdzdJUTbo8KFFGZEiaaSL5SM9h3AW6bKDIleqgF", + "yfRURrT7xxQcjiGoGIPgC9NGh5IG02VNo7qUUBZbrnltWS75EQshNZOTYswmuXIZvFYE+da6xwrqowda", + "87ijAkem2IwYGLIsws+/QydniFCqQNfOnrj3rL/fe/J0v9ft7ve62/iBNkStijEX9t03BJjHKwPMNssx", + "sEl/RcS8cIPdLJllK4WQ2a1k6G8IkhtliBaoYhWpJJg8CUXX29Sg7rPu5AtHQFExYndlp8ICts6aF4XB", + "LAXCouLsyA3+FG3kq1d0gC5PT1Ggjka5cWAsuAHaO+Yyp+j1PAM1Y1oqJIhhM3hkKZznQjAxsRQsvCWJ", + "fcPnSPnv108+I7n23O3czD2tn3ExzQ2Vn4Wbo6e5QfbJLdmKEBLSehLeMQborXRzwkpbSMjlzOaHE0FH", + "8+bw5Sy4lxCBRhZgayMV0Ed/igrOC5rGLRw0hlvYi49buJDKfvSrc58c48pOL9ypGi0bEMkVloc2SK+I", + "zUy4+oMbhy5Pq07xNOpjU7meoPQE7bA6sTi5TEkjE8lrlWNskqyiL/+U0ywi/5LHLFbXqsoe8xDvjU2V", + "keDdQyPX+M3JC3tOKcaugSYbw+Fdo9jus9sWJG6PvtYXmtf1Xn0T1L5bqb9qu/UbT7s/YFG7GssLJhuj", + "eCNjfGefm+miwV1z/PvodhdHhCbnde3vIusOYzYZdnWNTa46ECztxYJHa32H3RoEJLliZn5hk7jX+QiI", + "AnWUe5277O6EcF8vmE+NyfDNjSvKjyOx5DcQoFiCjs5O3LEpJYJMbJ68PEWcjSGZJxxQ7grojSTm+qrv", + "jk/a9jBAUQHS3fGRGacQOzolwtLHLTwDpT3f7n5v33WtZQaCZAwP8IH7qoWtGpyInWlZSZ6AMztrdC4W", + "nVC3dhNqzVazOpNCe930u11fehcm2CtZdF86f2lfDvKIaBNeChycCpec0aohcVblF+qxk87TlKi5ld19", + "i5IpJJ/cq47DT3qlQG+YNid+yHdKtBUU9AXzJv5rSGrXZZFrWP5NCx92e3emYd9Gi7D9XZDcTKViX4Fa", + "po/vcFtXMj0RBpQgHGlQM1ChKVJ1Qjz4UHe/Dx9vPlb33alroatM6sheV27NYB8YQJvnks7vTMTIvZyb", + "ehCy6eymYWn9O1tBMLCIkl0JZFRUID2qJ3oukkfeunaw0c8JRUVH9e+y6MPu4Q4seqmJ94A86SznHBFB", + "UahAL9oG1XjaubaQ5sYnNw7+qFn3thfu+8LbMqJICgaUditY2qPzN20QiaQWn3jVhcOvfetydIH4CihV", + "96hWRXHLEOBjw9sOVxUIvSg/zWQLM/G7WxhGayVa+I799/cSF9cSf+m/Co2oX/qvCM+YgF8Ojha3E+/H", + "WLq7Cs3FfZKfxrfR+H6DkOwXSnOhKRzSN6C9ctROAF/RDrkN5itX+BP2bQP7qupai/wWral7BH9Ld5e3", + "wn93t8ULe4spPBSxQvXhX4X7HopJeytyCMzB9EVDtR7jOteMboO/Fja/lIIj6dIVRu4aWRVGt3NwVTB+", + "kCnOteesEdAAtCp5ZCXW2uled3cbs3YOjx60+TiE1FBdM4B0uJzoddW+Qg1vpOvj37ldtRq3WyTn8jOy", + "60J72iggqS96Xly8LFH+VQ5qvuA5dnNwlc9ykbr5e6vV3WLOhP91gQKTK+GvRoG7ixvjHu4JR3j3Yu3q", + "LVzJwBfTgRkI0/YaqBtV5EKwnZBxwsT6kU3IKScosPjpWNvFZWeRpW95O3W2GXOv0Ah2/ZsoMj33A/7R", + "obvohv/NJnbYfXb/rI+lGHOWGNRe2IhdBRMWzgk6miOpqtcMHpLxB2NdSOYiY5Arav/Fu5X2H244/KPt", + "f7H3/3IPSKRSkBh/+ehhFcUrcKriynvuvtLiHlCrgOuXp6fxhBCujnWu/YeTTWe4xU/e7wl9RYgUS3sQ", + "XhYuB1AIN0p27mGyvOvwQAv5VnGFCC6gV8+a8ahd/VMMD8Eu777YF/tjFFuV+nbqFeU9qx/FK3adgcIa", + "CHc/k6np46E4qLe0QhIjlwqCldvKK1sel+V95ftveISgcIt2RyHBz8rwFs2OirLWtTrK0Hx/jY5viH13", + "t7mFla2MfD9bHD98i2NW7OEiim3Z1Lg/4LFVS6OEnLttaFz+OPmU6QeZSsOllVmZolZVvXdpYN3dBcVd", + "91AuH/C56Dcokm2lf+IIWIqxW0xvZEI4ojADLjP3414/Frdwrni4GD7o+D82MJXauB/H4JuPN/8LAAD/", + "/x8BGYD1TgAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index aed503c7..28c08e06 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -434,6 +434,12 @@ paths: application/json: schema: $ref: "#/components/schemas/Error" + 404: + description: Image not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" 401: description: Unauthorized content: From 8126152c5f7851f6bebb6464b88c50e98e9fdd94 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 15:44:51 -0500 Subject: [PATCH 30/37] Use docker auth config file --- cmd/api/api/images_test.go | 4 +++- lib/images/oci.go | 11 +++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/cmd/api/api/images_test.go b/cmd/api/api/images_test.go index 686cfbb9..06cba5ab 100644 --- a/cmd/api/api/images_test.go +++ b/cmd/api/api/images_test.go @@ -80,7 +80,9 @@ func TestCreateImage_Async(t *testing.T) { require.NoError(t, err) imgResp, ok := getResp.(oapi.GetImage200JSONResponse) - require.True(t, ok, "expected 200 response") + if !ok { + t.Fatalf("expected 200 response, got %T: %+v", getResp, getResp) + } currentImg := oapi.Image(imgResp) currentQueuePos := getQueuePos(currentImg.QueuePosition) diff --git a/lib/images/oci.go b/lib/images/oci.go index 003e87fd..ba018db7 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/layout" @@ -69,7 +70,10 @@ func (c *ociClient) inspectManifest(ctx context.Context, imageRef string) (strin // Head request to get manifest descriptor - no automatic retries // Rate limits return immediately with actual error - descriptor, err := remote.Head(ref, remote.WithContext(ctx)) + // Use system authentication (reads from ~/.docker/config.json, etc.) + descriptor, err := remote.Head(ref, + remote.WithContext(ctx), + remote.WithAuthFromKeychain(authn.DefaultKeychain)) if err != nil { return "", fmt.Errorf("fetch manifest: %w", err) } @@ -122,7 +126,10 @@ func (c *ociClient) pullToOCILayout(ctx context.Context, imageRef, layoutTag str } // Fetch image manifest from registry (lazy - doesn't download layers yet) - img, err := remote.Image(ref, remote.WithContext(ctx)) + // Use system authentication (reads from ~/.docker/config.json, etc.) + img, err := remote.Image(ref, + remote.WithContext(ctx), + remote.WithAuthFromKeychain(authn.DefaultKeychain)) if err != nil { // Rate limits fail here immediately during manifest fetch return fmt.Errorf("fetch image manifest: %w", err) From 866f855300f1cd7589d2de2b211f698cd1c7bdae Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 15:56:26 -0500 Subject: [PATCH 31/37] Add timeout on resolve --- lib/images/manager.go | 6 +++++- lib/images/oci.go | 7 +++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/images/manager.go b/lib/images/manager.go index 5430cce5..84e1f61d 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -73,7 +73,11 @@ func (m *manager) CreateImage(ctx context.Context, req CreateImageRequest) (*Ima } // Resolve to get digest (validates existence) - ref, err := normalized.Resolve(ctx, m.ociClient) + // Add a 2-second timeout to ensure fast failure on rate limits or errors + resolveCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + ref, err := normalized.Resolve(resolveCtx, m.ociClient) if err != nil { return nil, fmt.Errorf("resolve manifest: %w", err) } diff --git a/lib/images/oci.go b/lib/images/oci.go index ba018db7..736c03ab 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -68,9 +68,8 @@ func (c *ociClient) inspectManifest(ctx context.Context, imageRef string) (strin return "", fmt.Errorf("parse image reference: %w", err) } - // Head request to get manifest descriptor - no automatic retries - // Rate limits return immediately with actual error // Use system authentication (reads from ~/.docker/config.json, etc.) + // Default retry: only on network errors, max ~1.3s total descriptor, err := remote.Head(ref, remote.WithContext(ctx), remote.WithAuthFromKeychain(authn.DefaultKeychain)) @@ -125,13 +124,13 @@ func (c *ociClient) pullToOCILayout(ctx context.Context, imageRef, layoutTag str return fmt.Errorf("parse image reference: %w", err) } - // Fetch image manifest from registry (lazy - doesn't download layers yet) // Use system authentication (reads from ~/.docker/config.json, etc.) + // Default retry: only on network errors, max ~1.3s total img, err := remote.Image(ref, remote.WithContext(ctx), remote.WithAuthFromKeychain(authn.DefaultKeychain)) if err != nil { - // Rate limits fail here immediately during manifest fetch + // Rate limits fail here immediately (429 is not retried by default) return fmt.Errorf("fetch image manifest: %w", err) } From 4001e3af324db2cf1c0949d18e6d1ecb0d113fcd Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 15:58:23 -0500 Subject: [PATCH 32/37] Update README --- lib/images/README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/images/README.md b/lib/images/README.md index 95dfb270..4f01144e 100644 --- a/lib/images/README.md +++ b/lib/images/README.md @@ -5,21 +5,22 @@ Converts OCI images to bootable erofs disks for Cloud Hypervisor VMs. ## Architecture ``` -OCI Registry → containers/image → OCI Layout → umoci → rootfs/ → mkfs.erofs → disk.erofs +OCI Registry → go-containerregistry → OCI Layout → umoci → rootfs/ → mkfs.erofs → disk.erofs ``` ## Design Decisions -### Why containers/image? (oci.go) +### Why go-containerregistry? (oci.go) **What:** Pull OCI images from any registry (Docker Hub, ghcr.io, etc.) **Why:** -- Standard library used by Podman, Skopeo, Buildah +- Lightweight library from Google (used by ko, crane, etc.) - Works directly with registries (no daemon required) +- No automatic retries on rate limits (immediate error propagation) - Supports all registry authentication methods -**Alternative:** Docker API - requires Docker daemon running +**Alternative:** containers/image - has automatic retry logic that delays error reporting ### Why umoci? (oci.go) @@ -120,11 +121,11 @@ Validation via `github.com/distribution/reference`: ## Build Tags -Requires `-tags containers_image_openpgp` to avoid C dependency on gpgme. This is a build-time option of the containers/image project to select between gpgme C library with go bindings or the pure Go OpenPGP implementation (slightly slower but doesn't need external system dependency). +Requires `-tags containers_image_openpgp` for umoci dependency compatibility. ## Registry Authentication -containers/image automatically uses `~/.docker/config.json` for registry authentication. +go-containerregistry automatically uses `~/.docker/config.json` via `authn.DefaultKeychain`. ```bash # Login to Docker Hub (avoid rate limits) From 28e25897265724ad325656ebe43e4be5f9a5fe77 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 16:19:24 -0500 Subject: [PATCH 33/37] Poll by digest --- Makefile | 2 +- cmd/api/api/images_test.go | 43 +++++++++++++++++++++++++++++++------- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 0412ccc5..7f91c4e4 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ dev: $(AIR) # Run tests test: - go test -tags containers_image_openpgp -v -timeout 30s ./... + go test -tags containers_image_openpgp -v -timeout 10s ./... # Clean generated files and binaries clean: diff --git a/cmd/api/api/images_test.go b/cmd/api/api/images_test.go index 06cba5ab..168578d3 100644 --- a/cmd/api/api/images_test.go +++ b/cmd/api/api/images_test.go @@ -2,6 +2,7 @@ package api import ( "fmt" + "strings" "testing" "time" @@ -67,16 +68,22 @@ func TestCreateImage_Async(t *testing.T) { img := oapi.Image(acceptedResp) require.Equal(t, "docker.io/library/alpine:latest", img.Name) - t.Logf("Image created: name=%s, initial_status=%s, queue_position=%v", - img.Name, img.Status, img.QueuePosition) + require.NotEmpty(t, img.Digest, "digest should be populated immediately") + t.Logf("Image created: name=%s, digest=%s, initial_status=%s, queue_position=%v", + img.Name, img.Digest, img.Status, img.QueuePosition) - // Poll until ready + // Construct digest reference for polling: repository@digest + // GetImage expects format like "docker.io/library/alpine@sha256:..." + digestRef := "docker.io/library/alpine@" + img.Digest + t.Logf("Polling with digest reference: %s", digestRef) + + // Poll until ready using digest (tag symlink doesn't exist until status=ready) t.Log("Polling for completion...") lastStatus := img.Status lastQueuePos := getQueuePos(img.QueuePosition) for i := 0; i < 3000; i++ { - getResp, err := svc.GetImage(ctx, oapi.GetImageRequestObject{Name: img.Name}) + getResp, err := svc.GetImage(ctx, oapi.GetImageRequestObject{Name: digestRef}) require.NoError(t, err) imgResp, ok := getResp.(oapi.GetImage200JSONResponse) @@ -192,10 +199,11 @@ func TestCreateImage_Idempotent(t *testing.T) { require.True(t, ok, "expected 202 response") img1 := oapi.Image(accepted1) require.Equal(t, imageName, img1.Name) + require.NotEmpty(t, img1.Digest, "digest should be populated immediately") require.Equal(t, oapi.ImageStatus(images.StatusPending), img1.Status) require.NotNil(t, img1.QueuePosition, "should have queue position") require.Equal(t, 1, *img1.QueuePosition, "should be at position 1") - t.Logf("First call: status=%s, queue_position=%v", img1.Status, formatQueuePos(img1.QueuePosition)) + t.Logf("First call: name=%s, digest=%s, status=%s, queue_position=%v", img1.Name, img1.Digest, img1.Status, formatQueuePos(img1.QueuePosition)) // Second call immediately - should return existing with same queue position t.Log("Second CreateImage call (immediate duplicate)...") @@ -208,15 +216,34 @@ func TestCreateImage_Idempotent(t *testing.T) { require.True(t, ok, "expected 202 response") img2 := oapi.Image(accepted2) require.Equal(t, imageName, img2.Name) + require.Equal(t, img1.Digest, img2.Digest, "should have same digest") + + // Log actual status to see what's happening + t.Logf("Second call: digest=%s, status=%s, queue_position=%v, error=%v", + img2.Digest, img2.Status, formatQueuePos(img2.QueuePosition), img2.Error) + + // If it failed, we need to see why + if img2.Status == oapi.ImageStatus(images.StatusFailed) { + if img2.Error != nil { + t.Logf("Build failed with error: %s", *img2.Error) + } + t.Fatal("Build failed - this is the root cause of test failures") + } + require.Equal(t, oapi.ImageStatus(images.StatusPending), img2.Status) require.NotNil(t, img2.QueuePosition, "should have queue position") require.Equal(t, 1, *img2.QueuePosition, "should still be at position 1") - t.Logf("Second call: status=%s, queue_position=%v", img2.Status, formatQueuePos(img2.QueuePosition)) - // Wait for build to complete + // Construct digest reference: repository@digest + // Extract repository from imageName (strip tag part) + repository := strings.Split(imageName, ":")[0] + digestRef := repository + "@" + img1.Digest + t.Logf("Polling with digest reference: %s", digestRef) + + // Wait for build to complete - poll by digest (tag symlink doesn't exist until status=ready) t.Log("Waiting for build to complete...") for i := 0; i < 3000; i++ { - getResp, err := svc.GetImage(ctx, oapi.GetImageRequestObject{Name: imageName}) + getResp, err := svc.GetImage(ctx, oapi.GetImageRequestObject{Name: digestRef}) require.NoError(t, err) imgResp, ok := getResp.(oapi.GetImage200JSONResponse) From 3b09260a57f0bde3439459cc564a384b2f950bfe Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 16:33:09 -0500 Subject: [PATCH 34/37] Tweak readme --- lib/images/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/images/README.md b/lib/images/README.md index 4f01144e..ad698294 100644 --- a/lib/images/README.md +++ b/lib/images/README.md @@ -17,10 +17,10 @@ OCI Registry → go-containerregistry → OCI Layout → umoci → rootfs/ → m **Why:** - Lightweight library from Google (used by ko, crane, etc.) - Works directly with registries (no daemon required) -- No automatic retries on rate limits (immediate error propagation) +- Can propagate errors from registry (like 429) - Supports all registry authentication methods -**Alternative:** containers/image - has automatic retry logic that delays error reporting +**Alternative:** containers/image - has automatic retry logic that delays error reporting, can't fail fast for registry rate limits. Heavier, supporting more use cases in comparison to go-containerregistry. ### Why umoci? (oci.go) @@ -45,7 +45,7 @@ OCI Registry → go-containerregistry → OCI Layout → umoci → rootfs/ → m - No journal/inode overhead **Options:** -- `-zlz4` - Fast compression (good balance for development) +- `-zlz4` - Fast compression **Alternative:** ext4 without journal works but erofs is optimized for this exact use case From 55ccb76ddd326ba00eddcc20e8ae107a3a246832 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 16:48:53 -0500 Subject: [PATCH 35/37] caching test --- lib/images/manager_test.go | 77 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go index a9d31944..64e0af5b 100644 --- a/lib/images/manager_test.go +++ b/lib/images/manager_test.go @@ -3,6 +3,7 @@ package images import ( "context" "os" + "path/filepath" "strings" "testing" "time" @@ -236,6 +237,82 @@ func TestNormalizedRefParsing(t *testing.T) { } } +func TestLayerCaching(t *testing.T) { + dataDir := t.TempDir() + mgr, err := NewManager(dataDir, 1) + require.NoError(t, err) + ctx := context.Background() + + // 1. Pull alpine:latest by tag + t.Log("Pulling alpine:latest by tag...") + alpine1, err := mgr.CreateImage(ctx, CreateImageRequest{ + Name: "docker.io/library/alpine:latest", + }) + require.NoError(t, err) + require.NotEmpty(t, alpine1.Digest, "should have digest") + + // Wait for first pull to complete (poll by digest) + alpine1Ref := "docker.io/library/alpine@" + alpine1.Digest + waitForReady(t, mgr, ctx, alpine1Ref) + + // Count blobs after first pull + blobsDir := filepath.Join(dataDir, "system", "oci-cache", "blobs", "sha256") + blobsAfterFirst, err := countFiles(blobsDir) + require.NoError(t, err) + t.Logf("Blobs after first pull: %d", blobsAfterFirst) + require.Greater(t, blobsAfterFirst, 0, "should have downloaded blobs") + + // 2. Pull the SAME digest but reference it by digest + // This guarantees 100% layer overlap - tests cross-reference caching + t.Logf("Pulling same image by digest reference: %s", alpine1.Digest) + alpine2, err := mgr.CreateImage(ctx, CreateImageRequest{ + Name: alpine1Ref, // Pull by digest instead of tag + }) + require.NoError(t, err) + require.Equal(t, alpine1.Digest, alpine2.Digest, "should have same digest") + + // This should be instant - already cached + waitForReady(t, mgr, ctx, alpine1Ref) + + // Count blobs after second pull + blobsAfterSecond, err := countFiles(blobsDir) + require.NoError(t, err) + t.Logf("Blobs after second pull: %d", blobsAfterSecond) + + // 3. Verify layer caching worked - should add ZERO new blobs + blobsAdded := blobsAfterSecond - blobsAfterFirst + require.Equal(t, 0, blobsAdded, + "Pulling same digest with different reference should not download any new blobs (everything cached)") + + // 4. Verify both references work and point to functional images + alpine1Parsed, err := ParseNormalizedRef(alpine1.Name) + require.NoError(t, err) + alpine2Parsed, err := ParseNormalizedRef(alpine2.Name) + require.NoError(t, err) + + // Both should point to the same digest directory + digestHex := strings.TrimPrefix(alpine1.Digest, "sha256:") + disk1 := digestPath(dataDir, alpine1Parsed.Repository(), digestHex) + disk2 := digestPath(dataDir, alpine2Parsed.Repository(), digestHex) + + require.Equal(t, disk1, disk2, "both references should point to same disk") + + stat, err := os.Stat(disk1) + require.NoError(t, err) + require.Greater(t, stat.Size(), int64(0)) + + t.Logf("Layer caching verified: second pull reused all %d cached blobs", blobsAfterFirst) +} + +// countFiles counts the number of files in a directory +func countFiles(dir string) (int, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return 0, err + } + return len(entries), nil +} + // waitForReady waits for an image build to complete func waitForReady(t *testing.T, mgr Manager, ctx context.Context, imageName string) { for i := 0; i < 600; i++ { From 325b5e9d57723cb6e4413e70fef25dafbc5cfca0 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 10 Nov 2025 16:53:17 -0500 Subject: [PATCH 36/37] More comprehensive basic test --- Makefile | 2 +- lib/images/manager_test.go | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 7f91c4e4..0412ccc5 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ dev: $(AIR) # Run tests test: - go test -tags containers_image_openpgp -v -timeout 10s ./... + go test -tags containers_image_openpgp -v -timeout 30s ./... # Clean generated files and binaries clean: diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go index 64e0af5b..c844a074 100644 --- a/lib/images/manager_test.go +++ b/lib/images/manager_test.go @@ -40,14 +40,45 @@ func TestCreateImage(t *testing.T) { ref, err := ParseNormalizedRef(img.Name) require.NoError(t, err) digestHex := strings.SplitN(img.Digest, ":", 2)[1] + + // Check erofs disk file diskPath := digestPath(dataDir, ref.Repository(), digestHex) - _, err = os.Stat(diskPath) + diskStat, err := os.Stat(diskPath) require.NoError(t, err) + require.False(t, diskStat.IsDir(), "disk path should be a file") + require.Greater(t, diskStat.Size(), int64(1000000), "erofs disk should be at least 1MB") + require.Equal(t, diskStat.Size(), *img.SizeBytes, "disk size should match metadata") + t.Logf("EROFS disk: path=%s, size=%d bytes", diskPath, diskStat.Size()) - // Check that tag symlink exists + // Check metadata file + metadataPath := metadataPath(dataDir, ref.Repository(), digestHex) + metaStat, err := os.Stat(metadataPath) + require.NoError(t, err) + require.False(t, metaStat.IsDir(), "metadata should be a file") + + // Read and verify metadata content + meta, err := readMetadata(dataDir, ref.Repository(), digestHex) + require.NoError(t, err) + require.Equal(t, img.Name, meta.Name) + require.Equal(t, img.Digest, meta.Digest) + require.Equal(t, StatusReady, meta.Status) + require.Nil(t, meta.Error) + require.Equal(t, diskStat.Size(), meta.SizeBytes) + require.NotEmpty(t, meta.Env, "should have environment variables") + t.Logf("Metadata: name=%s, digest=%s, status=%s, env_vars=%d", + meta.Name, meta.Digest, meta.Status, len(meta.Env)) + + // Check that tag symlink exists and points to correct digest linkPath := tagSymlinkPath(dataDir, ref.Repository(), ref.Tag()) - _, err = os.Lstat(linkPath) + linkStat, err := os.Lstat(linkPath) + require.NoError(t, err) + require.NotEqual(t, 0, linkStat.Mode()&os.ModeSymlink, "should be a symlink") + + // Verify symlink points to digest directory + linkTarget, err := os.Readlink(linkPath) require.NoError(t, err) + require.Equal(t, digestHex, linkTarget, "symlink should point to digest") + t.Logf("Tag symlink: %s -> %s", linkPath, linkTarget) } func TestCreateImageDifferentTag(t *testing.T) { From f310f98aa5c11c5f1238acf81153752df0584713 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Tue, 11 Nov 2025 09:38:13 -0500 Subject: [PATCH 37/37] 404 error mapping --- cmd/api/api/images.go | 8 +++----- lib/images/errors.go | 22 +++++++++++++++++++++- lib/images/oci.go | 4 ++-- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/cmd/api/api/images.go b/cmd/api/api/images.go index 740d5746..b1cd4b60 100644 --- a/cmd/api/api/images.go +++ b/cmd/api/api/images.go @@ -3,8 +3,6 @@ package api import ( "context" "errors" - "fmt" - "strings" "github.com/onkernel/hypeman/lib/images" "github.com/onkernel/hypeman/lib/logger" @@ -46,10 +44,10 @@ func (s *ApiService) CreateImage(ctx context.Context, request oapi.CreateImageRe Code: "invalid_name", Message: err.Error(), }, nil - case strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "not found"): + case errors.Is(err, images.ErrNotFound): return oapi.CreateImage404JSONResponse{ Code: "not_found", - Message: fmt.Sprintf("image not found: %v", err), + Message: "image not found", }, nil default: log.Error("failed to create image", "error", err, "name", request.Body.Name) @@ -131,4 +129,4 @@ func imageToOAPI(img images.Image) oapi.Image { } return oapiImg -} \ No newline at end of file +} diff --git a/lib/images/errors.go b/lib/images/errors.go index 63b3bb3b..ee7679e8 100644 --- a/lib/images/errors.go +++ b/lib/images/errors.go @@ -1,8 +1,28 @@ package images -import "errors" +import ( + "errors" + "fmt" + "strings" +) var ( ErrNotFound = errors.New("image not found") ErrInvalidName = errors.New("invalid image name") ) + +// wrapRegistryError checks if the error is a registry 404 error and wraps it as ErrNotFound. +// go-containerregistry returns transport errors with specific codes for registry issues. +func wrapRegistryError(err error) error { + if err == nil { + return nil + } + errStr := err.Error() + if strings.Contains(errStr, "NAME_UNKNOWN") || + strings.Contains(errStr, "MANIFEST_UNKNOWN") || + strings.Contains(errStr, "404") || + strings.Contains(errStr, "not found") { + return fmt.Errorf("%w: %v", ErrNotFound, err) + } + return err +} diff --git a/lib/images/oci.go b/lib/images/oci.go index 736c03ab..645b174d 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -74,7 +74,7 @@ func (c *ociClient) inspectManifest(ctx context.Context, imageRef string) (strin remote.WithContext(ctx), remote.WithAuthFromKeychain(authn.DefaultKeychain)) if err != nil { - return "", fmt.Errorf("fetch manifest: %w", err) + return "", fmt.Errorf("fetch manifest: %w", wrapRegistryError(err)) } return descriptor.Digest.String(), nil @@ -131,7 +131,7 @@ func (c *ociClient) pullToOCILayout(ctx context.Context, imageRef, layoutTag str remote.WithAuthFromKeychain(authn.DefaultKeychain)) if err != nil { // Rate limits fail here immediately (429 is not retried by default) - return fmt.Errorf("fetch image manifest: %w", err) + return fmt.Errorf("fetch image manifest: %w", wrapRegistryError(err)) } // Open or create OCI layout directory