diff --git a/docs/config.md b/docs/config.md index cd32705..4d194f0 100644 --- a/docs/config.md +++ b/docs/config.md @@ -17,8 +17,9 @@ To change the default container runtime: ```yaml # ... -container: - runtime: kubernetes +runtime: + container: + default_runtime: kubernetes # ... ``` diff --git a/pkg/action/runtime.container.go b/pkg/action/runtime.container.go index d078ee0..ccb90f5 100644 --- a/pkg/action/runtime.container.go +++ b/pkg/action/runtime.container.go @@ -29,8 +29,27 @@ const ( containerFlagRebuildImage = "rebuild-image" containerFlagEntrypoint = "entrypoint" containerFlagExec = "exec" + containerRegistryURL = "registry-url" + containerRegistryType = "registry-type" + containerRegistryInsecure = "registry-insecure" ) +// RuntimeFlags stores container runtime opts. +type runtimeContainerFlags struct { + IsSetRemote bool + CopyBack bool + RemoveImg bool + NoCache bool + RebuildImage bool + Entrypoint string + EntrypointSet bool + Exec bool + VolumeFlags string + RegistryURL string + RegistryInsecure bool + RegistryType driver.KubernetesRegistry +} + type runtimeContainer struct { WithLogger WithTerm @@ -50,15 +69,7 @@ type runtimeContainer struct { nameprv ContainerNameProvider // Runtime flags - isSetRemote bool - copyBack bool - removeImg bool - noCache bool - rebuildImage bool - entrypoint string - entrypointSet bool - exec bool - volumeFlags string + rcf runtimeContainerFlags } // ContainerNameProvider provides an ability to generate a random container name @@ -156,6 +167,26 @@ func (c *runtimeContainer) GetFlags() *FlagsGroup { Type: jsonschema.Boolean, Default: false, }, + &DefParameter{ + Name: containerRegistryURL, + Title: "Kubernetes Images registry URL", + Type: jsonschema.String, + Default: "localhost:5000", + }, + &DefParameter{ + Name: containerRegistryType, + Title: "Kubernetes Images registry type", + Type: jsonschema.String, + Enum: []any{driver.RegistryNone, driver.RegistryInternal, driver.RegistryRemote}, + Default: driver.RegistryNone.String(), + }, + &DefParameter{ + Name: containerRegistryInsecure, + Title: "Insecure Registry", + Description: "Allow Kubernetes image builder push to insecure registry", + Type: jsonschema.Boolean, + Default: false, + }, } flags.AddDefinitions(definitions) @@ -184,32 +215,44 @@ func (c *runtimeContainer) SetFlags(input *Input) error { flags := input.GroupFlags(c.flags.GetName()) if v, ok := flags[containerFlagRemote]; ok { - c.isSetRemote = v.(bool) + c.rcf.IsSetRemote = v.(bool) } if v, ok := flags[containerFlagCopyBack]; ok { - c.copyBack = v.(bool) + c.rcf.CopyBack = v.(bool) } if r, ok := flags[containerFlagRemoveImage]; ok { - c.removeImg = r.(bool) + c.rcf.RemoveImg = r.(bool) } if nc, ok := flags[containerFlagNoCache]; ok { - c.noCache = nc.(bool) + c.rcf.NoCache = nc.(bool) } if rb, ok := flags[containerFlagRebuildImage]; ok { - c.rebuildImage = rb.(bool) + c.rcf.RebuildImage = rb.(bool) } if e, ok := flags[containerFlagEntrypoint]; ok && e != "" { - c.entrypointSet = true - c.entrypoint = e.(string) + c.rcf.EntrypointSet = true + c.rcf.Entrypoint = e.(string) } if ex, ok := flags[containerFlagExec]; ok { - c.exec = ex.(bool) + c.rcf.Exec = ex.(bool) + } + + if rt, ok := flags[containerRegistryType]; ok { + c.rcf.RegistryType = driver.RegistryFromString(rt.(string)) + } + + if rurl, ok := flags[containerRegistryURL]; ok { + c.rcf.RegistryURL = rurl.(string) + } + + if rin, ok := flags[containerRegistryInsecure]; ok { + c.rcf.RegistryInsecure = rin.(bool) } return nil @@ -241,13 +284,13 @@ func (c *runtimeContainer) Init(ctx context.Context, _ *Action) (err error) { if !c.isRemote() && c.isSELinuxEnabled(ctx) { // Check SELinux settings to allow reading the FS inside a container. // Use the lowercase z flag to allow concurrent actions access to the FS. - c.volumeFlags += ":z" + c.rcf.VolumeFlags += ":z" launchr.Term().Warning().Printfln( "SELinux is detected. The volumes will be mounted with the %q flags, which will relabel your files.\n"+ "This process may take time or potentially break existing permissions.", - c.volumeFlags, + c.rcf.VolumeFlags, ) - c.Log().Warn("using selinux flags", "flags", c.volumeFlags) + c.Log().Warn("using selinux flags", "flags", c.rcf.VolumeFlags) } return nil @@ -285,40 +328,14 @@ func (c *runtimeContainer) Execute(ctx context.Context, a *Action) (err error) { return errors.New("error on creating a container") } - // Remove the container after finish. - defer func() { - log.Debug("remove container after run") - errRm := c.crt.ContainerRemove(ctx, cid) - if errRm != nil { - log.Error("error on cleaning the running environment", "error", errRm) - } else { - log.Debug("container was successfully removed") - } - }() - - // Remove the used image if it was specified. + // Cleanup resources on finish. defer func() { - if !c.removeImg { - return - } - log.Debug("removing container image after run") - errImg := c.imageRemove(ctx, a) - if errImg != nil { - log.Error("failed to remove image", "error", errImg) - } else { - log.Debug("image was successfully removed") - } + c.cleanupRuntimeResources(ctx, cid, a, &runConfig) }() log = c.LogWith("container_id", cid) log.Debug("successfully created a container for an action") - // Copy working dirs to the container. - err = c.copyAllToContainer(ctx, cid, a) - if err != nil { - return err - } - if !runConfig.Streams.TTY { log.Debug("watching container signals") sigc := launchr.NotifySignals() @@ -392,10 +409,13 @@ func (c *runtimeContainer) Close() error { return c.crt.Close() } -func (c *runtimeContainer) imageRemove(ctx context.Context, a *Action) error { +func (c *runtimeContainer) imageRemove(ctx context.Context, a *Action, opts driver.ImageOptions) error { if crt, ok := c.crt.(driver.ContainerImageBuilder); ok { _, err := crt.ImageRemove(ctx, a.RuntimeDef().Container.Image, driver.ImageRemoveOptions{ - Force: true, + Force: true, + RegistryType: opts.RegistryType, + RegistryURL: opts.RegistryURL, + BuildContainerID: opts.BuildContainerID, }) return err } @@ -405,7 +425,7 @@ func (c *runtimeContainer) imageRemove(ctx context.Context, a *Action) error { func (c *runtimeContainer) isRebuildRequired(bi *driver.BuildDefinition) (bool, error) { // @todo test image cache resolution somehow. - if c.imgccres == nil || bi == nil || !c.rebuildImage { + if c.imgccres == nil || bi == nil || !c.rcf.RebuildImage { return false, nil } @@ -435,7 +455,7 @@ func (c *runtimeContainer) isRebuildRequired(bi *driver.BuildDefinition) (bool, return doRebuild, nil } -func (c *runtimeContainer) imageEnsure(ctx context.Context, a *Action) error { +func (c *runtimeContainer) imageEnsure(ctx context.Context, a *Action, createOpts *driver.ContainerDefinition) error { crt, ok := c.crt.(driver.ContainerImageBuilder) if !ok { return nil @@ -451,12 +471,10 @@ func (c *runtimeContainer) imageEnsure(ctx context.Context, a *Action) error { return err } - status, err := crt.ImageEnsure(ctx, driver.ImageOptions{ - Name: image, - Build: buildInfo, - NoCache: c.noCache, - ForceRebuild: forceRebuild, - }) + createOpts.ImageOptions.Build = buildInfo + createOpts.ImageOptions.ForceRebuild = forceRebuild + + status, err := crt.ImageEnsure(ctx, createOpts.ImageOptions) if err != nil { return err } @@ -501,17 +519,16 @@ func (c *runtimeContainer) imageEnsure(ctx context.Context, a *Action) error { } func (c *runtimeContainer) containerCreate(ctx context.Context, a *Action, createOpts *driver.ContainerDefinition) (string, error) { - var err error - if err = c.imageEnsure(ctx, a); err != nil { - return "", err - } + switch c.rtype { + case driver.Kubernetes: + return c.containerCreateKubernetes(ctx, a, createOpts) - cid, err := c.crt.ContainerCreate(ctx, *createOpts) - if err != nil { - return "", err - } + case driver.Docker: + fallthrough - return cid, nil + default: + return c.containerCreateDocker(ctx, a, createOpts) + } } func (c *runtimeContainer) createContainerDef(a *Action, cname string) driver.ContainerDefinition { @@ -521,25 +538,32 @@ func (c *runtimeContainer) createContainerDef(a *Action, cname string) driver.Co // Override an entrypoint if it was set in flags. var entrypoint []string - if c.entrypointSet { - entrypoint = []string{c.entrypoint} + if c.rcf.EntrypointSet { + entrypoint = []string{c.rcf.Entrypoint} } // Override Command with exec command. cmd := runDef.Container.Command - if c.exec { + if c.rcf.Exec { cmd = a.Input().ArgsPositional() } createOpts := driver.ContainerDefinition{ ContainerName: cname, Image: runDef.Container.Image, - Command: cmd, - WorkingDir: containerHostMount, - ExtraHosts: runDef.Container.ExtraHosts, - Env: runDef.Container.Env, - User: getCurrentUser(), - Entrypoint: entrypoint, + ImageOptions: driver.ImageOptions{ + Name: runDef.Container.Image, + NoCache: c.rcf.NoCache, + RegistryURL: c.rcf.RegistryURL, + RegistryType: c.rcf.RegistryType, + RegistryInsecure: c.rcf.RegistryInsecure, + }, + Command: cmd, + WorkingDir: containerHostMount, + ExtraHosts: runDef.Container.ExtraHosts, + Env: runDef.Container.Env, + User: getCurrentUser(), + Entrypoint: entrypoint, Streams: driver.ContainerStreamsOptions{ Stdin: !streams.In().IsDiscard(), Stdout: !streams.Out().IsDiscard(), @@ -556,8 +580,8 @@ func (c *runtimeContainer) createContainerDef(a *Action, cname string) driver.Co ) } else { createOpts.Binds = []string{ - launchr.MustAbs(a.WorkDir()) + ":" + containerHostMount + c.volumeFlags, - launchr.MustAbs(a.Dir()) + ":" + containerActionMount + c.volumeFlags, + launchr.MustAbs(a.WorkDir()) + ":" + containerHostMount + c.rcf.VolumeFlags, + launchr.MustAbs(a.Dir()) + ":" + containerActionMount + c.rcf.VolumeFlags, } } return createOpts @@ -591,7 +615,7 @@ func (c *runtimeContainer) copyAllToContainer(ctx context.Context, cid string, a } func (c *runtimeContainer) copyAllFromContainer(ctx context.Context, cid string, a *Action) (err error) { - if !c.isRemote() || !c.copyBack { + if !c.isRemote() || !c.rcf.CopyBack { return nil } // @todo it's a bad implementation considering consequential runs, need to find a better way to sync with remote. @@ -670,5 +694,103 @@ func (c *runtimeContainer) isSELinuxEnabled(ctx context.Context) bool { } func (c *runtimeContainer) isRemote() bool { - return c.isRemoteRuntime || c.isSetRemote + return c.isRemoteRuntime || c.rcf.IsSetRemote +} + +func (c *runtimeContainer) containerCreateDocker(ctx context.Context, a *Action, createOpts *driver.ContainerDefinition) (string, error) { + var err error + if err = c.imageEnsure(ctx, a, createOpts); err != nil { + return "", err + } + + cid, err := c.crt.ContainerCreate(ctx, *createOpts) + if err != nil { + return "", err + } + + // Copy working dirs to the container. + err = c.copyAllToContainer(ctx, cid, a) + if err != nil { + return "", err + } + + return cid, nil +} + +func (c *runtimeContainer) containerCreateKubernetes(ctx context.Context, a *Action, createOpts *driver.ContainerDefinition) (string, error) { + var err error + cid, err := c.crt.ContainerCreate(ctx, *createOpts) + if err != nil { + return "", err + } + + // Set the build container ID to the image options. + createOpts.ImageOptions.BuildContainerID = driver.K8SPodBuildContainerID(cid) + + // Copy working dirs to the container. + err = c.copyAllToContainer(ctx, cid, a) + if err != nil { + return "", err + } + + if err = c.imageEnsure(ctx, a, createOpts); err != nil { + return "", err + } + + return cid, nil +} + +// cleanupRuntimeResources handles cleanup in the correct order for each runtime type +func (c *runtimeContainer) cleanupRuntimeResources(ctx context.Context, cid string, a *Action, createOpts *driver.ContainerDefinition) { + switch c.rtype { + case driver.Kubernetes: + c.cleanupKubernetesResources(ctx, cid, a, createOpts.ImageOptions) + case driver.Docker: + fallthrough + default: + //panic(fmt.Errorf("unsupported runtime type for cleanup: %s", c.rtype)) + c.cleanupDockerResources(ctx, cid, a, createOpts.ImageOptions) + } +} + +// cleanupDockerResources handles Docker-specific cleanup order +func (c *runtimeContainer) cleanupDockerResources(ctx context.Context, cid string, a *Action, options driver.ImageOptions) { + // Remove the container first + c.Log().Debug("remove container after run") + if err := c.crt.ContainerRemove(ctx, cid); err != nil { + c.Log().Error("failed to remove container", "error", err) + } else { + c.Log().Debug("container was successfully removed") + } + + // Then remove the image if requested + if c.rcf.RemoveImg { + c.Log().Debug("removing container image after run") + if err := c.imageRemove(ctx, a, options); err != nil { + c.Log().Error("failed to remove image", "error", err) + } else { + c.Log().Debug("image was successfully removed") + } + } +} + +// cleanupKubernetesResources handles Kubernetes-specific cleanup order +func (c *runtimeContainer) cleanupKubernetesResources(ctx context.Context, cid string, a *Action, options driver.ImageOptions) { + // Remove image first if requested + if c.rcf.RemoveImg { + c.Log().Debug("removing container image after run") + if err := c.imageRemove(ctx, a, options); err != nil { + c.Log().Error("failed to remove image", "error", err) + } else { + c.Log().Debug("image was successfully removed") + } + } + + // Then remove the pod / container + c.Log().Debug("remove container after run") + if err := c.crt.ContainerRemove(ctx, cid); err != nil { + c.Log().Error("failed to remove container", "error", err) + } else { + c.Log().Debug("Container was successfully removed") + } } diff --git a/pkg/action/runtime.container_test.go b/pkg/action/runtime.container_test.go index 04af250..4f4c78a 100644 --- a/pkg/action/runtime.container_test.go +++ b/pkg/action/runtime.container_test.go @@ -192,7 +192,10 @@ func Test_ContainerExec_imageEnsure(t *testing.T) { d.EXPECT(). ImageEnsure(ctx, eqImageOpts{imgOpts}). Return(tt.ret...) - err := r.imageEnsure(ctx, act) + + cname := launchr.GetRandomString(4) + runCfg := r.createContainerDef(act, cname) + err := r.imageEnsure(ctx, act, &runCfg) assert.Equal(t, tt.ret[1], err) }) } @@ -248,7 +251,7 @@ func Test_ContainerExec_imageRemove(t *testing.T) { d.EXPECT(). ImageRemove(ctx, run.Image, gomock.Eq(imgOpts)). Return(tt.ret...) - err := r.imageRemove(ctx, act) + err := r.imageRemove(ctx, act, driver.ImageOptions{Name: run.Image}) assert.Equal(t, err, tt.ret[1]) }) @@ -265,7 +268,10 @@ func Test_ContainerExec_createContainerDef(t *testing.T) { } baseRes := driver.ContainerDefinition{ - Image: "myimage", + Image: "myimage", + ImageOptions: driver.ImageOptions{ + Name: "myimage", + }, WorkingDir: containerHostMount, ExtraHosts: []string{ "my:host1", @@ -342,9 +348,11 @@ func Test_ContainerExec_createContainerDef(t *testing.T) { input.SetValidated(true) _ = a.SetInput(input) r.isRemoteRuntime = true - r.entrypointSet = true - r.entrypoint = "/my/entrypoint" - r.exec = true + r.rcf = runtimeContainerFlags{ + EntrypointSet: true, + Entrypoint: "/my/entrypoint", + Exec: true, + } return a }, driver.ContainerDefinition{ @@ -371,6 +379,7 @@ func Test_ContainerExec_createContainerDef(t *testing.T) { a.SetRuntime(r) cname := tt.exp.ContainerName tt.exp.Image = baseRes.Image + tt.exp.ImageOptions = baseRes.ImageOptions tt.exp.WorkingDir = baseRes.WorkingDir tt.exp.ExtraHosts = baseRes.ExtraHosts tt.exp.Env = baseRes.Env @@ -409,6 +418,7 @@ func Test_ContainerExec(t *testing.T) { ContainerName: nprv.Get(act.ID), Command: runConf.Command, Image: runConf.Image, + ImageOptions: driver.ImageOptions{Name: runConf.Image}, ExtraHosts: runConf.ExtraHosts, Binds: []string{ launchr.MustAbs(act.WorkDir()) + ":" + containerHostMount, diff --git a/pkg/driver/k8s.templates.go b/pkg/driver/k8s.templates.go new file mode 100644 index 0000000..db498b9 --- /dev/null +++ b/pkg/driver/k8s.templates.go @@ -0,0 +1,186 @@ +package driver + +import ( + "fmt" + "strings" + + "github.com/launchrctl/launchr/internal/launchr" +) + +func executeTemplate(templateStr string, data any) (string, error) { + tpl := launchr.Template{ + Tmpl: templateStr, + Data: data, + } + var buf strings.Builder + + err := tpl.Generate(&buf) + if err != nil { + return "", err + } + + return buf.String(), nil +} + +// Registry removal template +const registryImageRemovalTemplate = ` +set -e +cd /action + +echo "Removing image {{.ImageName}}:{{.Tag}} from registry" + +# Get the digest of the image +DIGEST=$(curl -s -I -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ +-H "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \ +-H "Accept: application/vnd.oci.image.manifest.v1+json" \ +-H "Accept: application/vnd.oci.image.index.v1+json" \ +{{.RegistryURL}}/v2/{{.ImageName}}/manifests/{{.Tag}} | \ +grep -i 'docker-content-digest' | \ +awk -F': ' '{print $2}' | \ +sed 's/[\r\n]//g') + +if [ -n "$DIGEST" ] && [ "$DIGEST" != "null" ]; then + echo "Found digest: $DIGEST" + # Delete the manifest + curl -X DELETE "{{.RegistryURL}}/v2/{{.ImageName}}/manifests/$DIGEST" || true + echo "Image removed from registry" +else + echo "Image not found in registry or already removed" +fi +` + +// ensure image exists in local registry +const buildahImageEnsureTemplate = ` +set -e +cd /action + +image_status=$(curl -s -I -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ +-H "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \ +-H "Accept: application/vnd.oci.image.manifest.v1+json" \ +-H "Accept: application/vnd.oci.image.index.v1+json" \ +-o /dev/null -w "%%{http_code}" %s) +if [ "$image_status" = "200" ]; then + exit 0 +else + exit 2 +fi +` + +const buildahInitTemplate = ` +set -e +echo "Buildah sidecar started" + +# Create containers config directory +mkdir -p /etc/containers + +# Configure registries +cat > /etc/containers/registries.conf << 'EOF' +unqualified-search-registries = ["docker.io"] + +[[registry]] +location = "{{.RegistryURL}}" +{{if .Insecure}}insecure = true{{else}}insecure = false{{end}} +EOF + +# Test registry connectivity +curl -f {{.RegistryURL}}/v2/ || { + echo "ERROR: Cannot connect to registry" + exit 1 +} +echo 'Registry connection works' + +# Test buildah +buildah version + +# Keep running to maintain container availability +while true; do + sleep 60 + echo "Buildah sidecar still running..." +done +` + +// Buildah image build template +const buildahBuildTemplate = ` +set -e +cd /action + +echo "Starting image build process..." +# Build the image +buildah build --layers \ + -t {{.RegistryURL}}/{{.ImageName}} \ + -f {{.Buildfile}} \ +{{- range $key, $value := .BuildArgs}} + --build-arg {{$key}}="{{$value}}" \ +{{- end}} + . 2>&1 + +if [ $? -ne 0 ]; then + echo "ERROR: Build failed with exit code $?" + exit 1 +fi + +echo "Build completed successfully" + +# Push to registry +echo "Pushing image to registry..." +buildah push {{.RegistryURL}}/{{.ImageName}} 2>&1 + +if [ $? -ne 0 ]; then + echo "ERROR: Push failed with exit code $?" + exit 1 +fi + +echo "Build and push completed successfully!" +echo "Image available at: {{.RegistryURL}}/{{.ImageName}}" +` + +func (k *k8sRuntime) prepareBuildahInitScript(opts ImageOptions) string { + type buildData struct { + ImageName string + RegistryURL string + Insecure bool + } + + data := &buildData{ + RegistryURL: opts.RegistryURL, + Insecure: opts.RegistryInsecure, + } + + script, err := executeTemplate(buildahInitTemplate, data) + if err != nil { + panic(fmt.Sprintf("failed to generate init build script: %s", err.Error())) + } + + return script +} + +func (k *k8sRuntime) prepareBuildahWorkScript(imageName string, opts ImageOptions) string { + // Add build args to template data + type BuildData struct { + ImageName string + RegistryURL string + BuildArgs map[string]*string + Buildfile string + Insecure bool + } + + buildFile := "Dockerfile" + if opts.Build.Buildfile != "" { + buildFile = opts.Build.Buildfile + } + + buildData := &BuildData{ + BuildArgs: opts.Build.Args, + Buildfile: buildFile, + ImageName: imageName, + Insecure: opts.RegistryInsecure, + RegistryURL: opts.RegistryURL, + } + + script, err := executeTemplate(buildahBuildTemplate, buildData) + if err != nil { + panic(fmt.Sprintf("failed to generate init build script: %s", err.Error())) + } + + return script +} diff --git a/pkg/driver/k8s.utils.go b/pkg/driver/k8s.utils.go index d9da1b9..1df095d 100644 --- a/pkg/driver/k8s.utils.go +++ b/pkg/driver/k8s.utils.go @@ -62,6 +62,12 @@ func k8sCreateContainerID(namespace, podName, containerName string) string { return namespace + "/" + podName + "/" + containerName } +// K8SPodBuildContainerID returns the image build container ID from the given container ID. +func K8SPodBuildContainerID(cid string) string { + namespace, podName, _ := k8sParseContainerID(cid) + return k8sCreateContainerID(namespace, podName, k8sBuildPodContainer) +} + func k8sPodMainContainerID(cid string) string { namespace, podName, _ := k8sParseContainerID(cid) return k8sCreateContainerID(namespace, podName, k8sMainPodContainer) @@ -257,3 +263,20 @@ func fillFileStatFromSys(modeHex uint32) os.FileMode { } return mode } + +func parseImageName(image string) (string, string) { + var name string + tag := "latest" + + nameParts := strings.Split(image, ":") + if len(nameParts) == 1 { + // Only image name, no tag or port + name = image + } else if len(nameParts) > 1 { + // Image name and tag + name = nameParts[0] + tag = nameParts[1] + } + + return name, tag +} diff --git a/pkg/driver/kubernetes.go b/pkg/driver/kubernetes.go index e0d764b..6707cb1 100644 --- a/pkg/driver/kubernetes.go +++ b/pkg/driver/kubernetes.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "k8s.io/apimachinery/pkg/api/resource" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -25,7 +27,42 @@ import ( "github.com/launchrctl/launchr/internal/launchr" ) +// KubernetesRegistry defines a registry type for a kubernetes container. +type KubernetesRegistry string + +// String returns the string representation of the registry type. +func (k KubernetesRegistry) String() string { + return string(k) +} + +// RegistryInternal defines an internal registry type +const RegistryInternal KubernetesRegistry = "internal" + +// RegistryRemote defines a remote registry type +const RegistryRemote KubernetesRegistry = "remote" + +// RegistryNone defines no registry type +const RegistryNone KubernetesRegistry = "none" + +// RegistryFromString creates a [KubernetesRegistry] with enum validation. +func RegistryFromString(t string) KubernetesRegistry { + if t == "" { + return RegistryNone + } + switch KubernetesRegistry(t) { + case RegistryInternal, RegistryRemote, RegistryNone: + return KubernetesRegistry(t) + default: + return RegistryNone + } +} + const k8sMainPodContainer = "supervisor" +const k8sBuildPodContainer = "image-builder" + +// Image build and image consist of multiple layers, artifacts, intermediate containers, and buildah needs temporary +// space to store these during the build process. 2Gi provides enough space for most typical application images. +const k8sBuildahStorageLimit = "2Gi" const k8sUseWebsocket = true const k8sStatPathScript = ` FILE="%s" @@ -207,6 +244,11 @@ func (k *k8sRuntime) ContainerCreate(ctx context.Context, opts ContainerDefiniti hostAliases := k8sHostAliases(opts) volumes, mounts := k8sVolumesAndMounts(opts) + sidecars, volumes, mounts, err := k.prepareSidecarContainers(volumes, mounts, opts) + if err != nil { + return "", err + } + // Create the pod definition. pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -214,10 +256,12 @@ func (k *k8sRuntime) ContainerCreate(ctx context.Context, opts ContainerDefiniti Namespace: namespace, }, Spec: corev1.PodSpec{ - HostAliases: hostAliases, - Hostname: opts.Hostname, - RestartPolicy: corev1.RestartPolicyNever, - Volumes: volumes, + HostAliases: hostAliases, + Hostname: opts.Hostname, + HostNetwork: true, + RestartPolicy: corev1.RestartPolicyNever, + Volumes: volumes, + InitContainers: sidecars, Containers: []corev1.Container{ { Name: k8sMainPodContainer, @@ -232,7 +276,7 @@ func (k *k8sRuntime) ContainerCreate(ctx context.Context, opts ContainerDefiniti // Create the pod launchr.Log().Debug("creating pod", "namespace", namespace, "pod", podName) - _, err := k.clientset.CoreV1(). + _, err = k.clientset.CoreV1(). Pods(namespace). Create(ctx, pod, metav1.CreateOptions{}) if err != nil { @@ -253,11 +297,10 @@ func (k *k8sRuntime) ContainerCreate(ctx context.Context, opts ContainerDefiniti } launchr.Log().Debug("pod is running", "namespace", namespace, "pod", podName) - return cid, err + return cid, nil } func (k *k8sRuntime) ContainerStart(ctx context.Context, cid string, opts ContainerDefinition) (<-chan int, *ContainerInOut, error) { - // Create an ephemeral container to run. err := k.addEphemeralContainer(ctx, cid, opts) if err != nil { return nil, nil, err @@ -457,10 +500,19 @@ func (k *k8sRuntime) addEphemeralContainer(ctx context.Context, cid string, opts cmd := slices.Concat(opts.Entrypoint, opts.Command) + imageName := opts.Image + pullPolicy := corev1.PullIfNotPresent + if opts.ImageOptions.Build != nil && opts.ImageOptions.RegistryType != RegistryNone { + // always pull to skip the kubernetes internal cache. + pullPolicy = corev1.PullAlways + imageName = fmt.Sprintf("%s/%s", opts.ImageOptions.RegistryURL, opts.Image) + } + ephemeralContainer := corev1.EphemeralContainer{ EphemeralContainerCommon: corev1.EphemeralContainerCommon{ - Name: containerName, - Image: opts.Image, + Name: containerName, + Image: imageName, + ImagePullPolicy: pullPolicy, // Wrap the command into a script that will wait until a special signal USR1. // We do that to not miss any output before the attach. See ContainerStart. Command: []string{"/bin/sh", "-c", k8sWaitAttachScript, "--"}, @@ -525,3 +577,188 @@ func (k *k8sRuntime) addEphemeralContainer(ctx context.Context, cid string, opts return false, nil }) } + +func (k *k8sRuntime) prepareSidecarContainers(volumes []corev1.Volume, mounts []corev1.VolumeMount, opts ContainerDefinition) ([]corev1.Container, []corev1.Volume, []corev1.VolumeMount, error) { + regType := opts.ImageOptions.RegistryType + var containers []corev1.Container + + if regType == RegistryNone { + return containers, volumes, mounts, nil + } + + if regType == RegistryInternal { + // @todo would be great to implement internal type which includes registry as sidecar and builds everything + // inside pod. Init containers don't share network between main container and sidecar containers, + // so probably we should combine main and sidecar containers together. + panic("registry internal is not supported yet") + } + + buildahVolumes := []corev1.Volume{ + { + Name: "buildah-storage", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + SizeLimit: &[]resource.Quantity{resource.MustParse(k8sBuildahStorageLimit)}[0], + }, + }, + }, + } + buildahMounts := []corev1.VolumeMount{ + { + Name: "buildah-storage", + MountPath: "/var/lib/containers", + }, + } + + volumes = append(volumes, buildahVolumes...) + mounts = append(mounts, buildahMounts...) + + securityContext := &corev1.SecurityContext{ + Privileged: &[]bool{true}[0], + RunAsUser: &[]int64{0}[0], + } + sidecarPolicy := corev1.ContainerRestartPolicyAlways + + buildahContainer := corev1.Container{ + Name: k8sBuildPodContainer, + Image: "quay.io/buildah/stable:latest", // @todo change latest to a specific version ? + SecurityContext: securityContext, + RestartPolicy: &sidecarPolicy, + Command: []string{"/bin/bash"}, + Args: []string{ + "-c", + k.prepareBuildahInitScript(opts.ImageOptions), + }, + VolumeMounts: mounts, + Env: []corev1.EnvVar{ + { + Name: "STORAGE_DRIVER", + Value: "vfs", + }, + { + Name: "BUILDAH_ISOLATION", + Value: "chroot", + }, + }, + } + + containers = append(containers, buildahContainer) + + return containers, volumes, mounts, nil +} + +func (k *k8sRuntime) ImageEnsure(ctx context.Context, opts ImageOptions) (*ImageStatusResponse, error) { + if opts.RegistryType == RegistryNone || opts.Build == nil { + return &ImageStatusResponse{Status: ImagePull}, nil + } + + exists, err := k.doImageEnsure(ctx, opts) + if err != nil { + return &ImageStatusResponse{Status: ImageUnexpectedError}, err + } + + if exists && !opts.ForceRebuild && !opts.NoCache { + return &ImageStatusResponse{Status: ImagePull}, nil + } + + err = k.doImageBuild(ctx, opts) + if err != nil { + return &ImageStatusResponse{Status: ImageUnexpectedError}, err + } + + return &ImageStatusResponse{Status: ImageBuild}, nil +} + +func (k *k8sRuntime) doImageEnsure(ctx context.Context, opts ImageOptions) (bool, error) { + imageName, tag := parseImageName(opts.Name) + imageURL := fmt.Sprintf("%s/v2/%s/manifests/%s", opts.RegistryURL, imageName, tag) + imageCheckScript := fmt.Sprintf(buildahImageEnsureTemplate, imageURL) + cmdArr := []string{ + "/bin/bash", "-c", + imageCheckScript, + } + + var stdout bytes.Buffer + var exitCode int + err := k.containerExec(ctx, opts.BuildContainerID, cmdArr, k8sStreams{ + out: &stdout, + opts: ContainerStreamsOptions{ + Stdout: true, + }, + }) + + if err == nil { + return true, nil + } + + // Check if it's a CodeExitError that contains the exit status + var exitErr exec.CodeExitError + if errors.As(err, &exitErr) { + exitCode = exitErr.ExitStatus() + } + + // If the exit code is 2 (manually returned code), it means that the image does not exist. + if exitCode == 2 { + return false, nil + } + + return false, fmt.Errorf("error container exec: %w, message: %s", err, stdout.String()) +} + +func (k *k8sRuntime) doImageBuild(ctx context.Context, opts ImageOptions) error { + cmdArr := []string{ + "/bin/bash", "-c", + k.prepareBuildahWorkScript(opts.Name, opts), + } + + var stdout bytes.Buffer + err := k.containerExec(ctx, opts.BuildContainerID, cmdArr, k8sStreams{ + out: &stdout, + opts: ContainerStreamsOptions{ + Stdout: true, + }, + }) + + launchr.Log().Debug("build output: ", "output", stdout.String()) + if err != nil { + return fmt.Errorf("error container exec: %w", err) + } + + return nil +} + +func (k *k8sRuntime) ImageRemove(ctx context.Context, image string, removeOpts ImageRemoveOptions) (*ImageRemoveResponse, error) { + if removeOpts.RegistryType != RegistryRemote { + return &ImageRemoveResponse{Status: ImageUnexpectedError}, nil + } + + imageName, tag := parseImageName(image) + type removeData struct { + ImageName string + RegistryURL string + Tag string + } + data := &removeData{ + Tag: tag, + ImageName: imageName, + RegistryURL: removeOpts.RegistryURL, + } + + script, err := executeTemplate(registryImageRemovalTemplate, data) + if err != nil { + panic(fmt.Errorf("failed to generate registry removal script: %s", err.Error())) + } + + var stdout bytes.Buffer + err = k.containerExec(ctx, removeOpts.BuildContainerID, []string{"/bin/bash", "-c", script}, k8sStreams{ + out: &stdout, + opts: ContainerStreamsOptions{Stdout: true}, + }) + + launchr.Log().Info("registry removal output", "output", stdout.String()) + if err != nil { + return &ImageRemoveResponse{Status: ImageUnexpectedError}, fmt.Errorf("failed to remove image from registry: %w", err) + } + + return &ImageRemoveResponse{Status: ImageRemoved}, nil +} diff --git a/pkg/driver/type.go b/pkg/driver/type.go index bbe4412..16c474c 100644 --- a/pkg/driver/type.go +++ b/pkg/driver/type.go @@ -88,11 +88,19 @@ type ImageOptions struct { Build *BuildDefinition NoCache bool ForceRebuild bool + + RegistryType KubernetesRegistry + RegistryURL string + RegistryInsecure bool + BuildContainerID string } // ImageRemoveOptions stores options for removing an image. type ImageRemoveOptions struct { - Force bool + Force bool + RegistryType KubernetesRegistry + RegistryURL string + BuildContainerID string } // ImageStatus defines image status on local machine. @@ -105,6 +113,7 @@ const ( ImagePull // ImagePull - image is being pulled from the registry. ImageBuild // ImageBuild - image is being built. ImageRemoved // ImageRemoved - image was removed + ImagePostpone // ImagePostpone - image action was postponed ) // SystemInfo holds information about the container runner environment. @@ -186,6 +195,7 @@ type ContainerDefinition struct { Hostname string ContainerName string Image string + ImageOptions ImageOptions Entrypoint []string Command []string