Fix pipeline to have better helper image and error handling

.github/workflows/_build-builder.yml → .github/workflows/_build-helper.yml

@@ -1,4 +1,4 @@
-name: Build and Push Builder (Reusable)
+name: Build and Push Helper (Reusable)
 
 on:
     workflow_call:
@@ -46,13 +46,13 @@ jobs:
                   username: ${{ github.actor }}
                   password: ${{ secrets.GITHUB_TOKEN }}
 
-            - name: Extract metadata (tags, labels) for Builder
+            - name: Extract metadata (tags, labels) for Helper
               id: meta
               uses: docker/metadata-action@v5
               with:
                   images: |
-                      ghcr.io/${{ github.repository }}/builder
-                      ${{ secrets.DOCKERHUB_USERNAME }}/builder
+                      ghcr.io/${{ github.repository }}/helper
+                      ${{ secrets.DOCKERHUB_USERNAME }}/helper
                   tags: |
                       # on git tag push, create a tag with the version number (e.g., v1.2.3)
                       type=ref,event=tag
@@ -61,11 +61,11 @@ jobs:
                       # create a tag with the git sha for every push
                       type=sha
 
-            - name: Build and push Builder Plugin
+            - name: Build and push Helper Plugin
               uses: docker/build-push-action@v5
               with:
-                  context: ./plugins/builder
-                  file: ./plugins/builder/Dockerfile
+                  context: ./plugins/helper
+                  file: ./plugins/helper/Dockerfile
                   push: ${{ inputs.push-images }}
                   tags: ${{ steps.meta.outputs.tags }}
                   labels: ${{ steps.meta.outputs.labels }}

.github/workflows/cloudness-builder.yml → .github/workflows/cloudness-helper.yml

@@ -1,4 +1,4 @@
-name: Plugin - Builder
+name: Plugin - Helper
 
 on:
   # push:
@@ -15,12 +15,12 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  build-cloudness-builder:
-    name: Builder Plugin
+  build-cloudness-helper:
+    name: Helper Plugin
     permissions:
       contents: read
       packages: write
-    uses: ./.github/workflows/_build-builder.yml
+    uses: ./.github/workflows/_build-helper.yml
     secrets:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/cloudness-release.yml

@@ -24,12 +24,12 @@ jobs:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
-  build-cloudness-builder:
-    name: Builder Plugin
+  build-cloudness-helper:
+    name: Helper Plugin
     permissions:
       contents: read
       packages: write
-    uses: ./.github/workflows/_build-builder.yml
+    uses: ./.github/workflows/_build-helper.yml
     with:
       enable-cache: true
     secrets:
@@ -38,7 +38,7 @@ jobs:
 
   upload-scripts:
     name: Upload Scripts
-    needs: [build-cloudness-app, build-cloudness-builder]
+    needs: [build-cloudness-app, build-cloudness-helper]
     permissions:
       contents: read
     uses: ./.github/workflows/_upload-scripts.yml

.github/workflows/pr-validation.yml

@@ -18,16 +18,3 @@ jobs:
         secrets:
             DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
             DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-
-    validate-builder:
-        name: Validate Builder
-        permissions:
-            contents: read
-            packages: write
-        uses: ./.github/workflows/_build-builder.yml
-        with:
-            enable-cache: true
-            push-images: false
-        secrets:
-            DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-            DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

app/pipeline/convert/build_step.go

@@ -5,7 +5,6 @@ import (
 	"strings"
 
 	"github.com/cloudness-io/cloudness/app/pipeline"
-	"github.com/cloudness-io/cloudness/app/services/config"
 	specSvc "github.com/cloudness-io/cloudness/app/services/spec"
 	"github.com/cloudness-io/cloudness/types"
 	"github.com/cloudness-io/cloudness/types/enum"
@@ -22,136 +21,53 @@ func buildCommandNew(
 		return nil
 	}
 
-	switch specSvc.GetBuilder(spec) {
-	case enum.BuilderTypeDockerfile:
-		updateDockerFileBuildStep(in, pCtx, spec, step, buildVars)
-	case enum.BuilderTypeNixpacks:
-		updateNixpacksBuildStep(in, pCtx, spec, step, buildVars)
-	}
-
-	addImageSecrets(in, pCtx, step)
-	addSecrets(pCtx, step, buildVars)
-
-	return nil
-}
-
-func updateDockerFileBuildStep(
-	in *pipeline.RunnerContextInput,
-	pCtx *pipeline.RunnerContext,
-	spec *types.ApplicationSpec,
-	step *pipeline.Step,
-	buildVars map[string]string,
-) {
-	var cmd strings.Builder
 	gitSource := spec.Build.Source.Git
 	sourcePath := wsBuildVolumePath
 	if gitSource.BasePath != "" && gitSource.BasePath != "/" {
 		sourcePath = wsBuildVolumePath + gitSource.BasePath
 	}
-	image, _, cacheImage := specSvc.GetImage(in.Application, in.Deployment, in.Config)
 
-	addBuildKitConfig(&cmd, in.Config)
+	image, _, cacheImage := specSvc.GetImage(in.Application, in.Deployment, in.Config)
 
-	// Construct the buildctl-daemonless.sh command
-	cmd.WriteString(fmt.Sprintf(`buildctl-daemonless.sh build \
-        --frontend=dockerfile.v0 \
-        --local context=%[1]s \
-        --local dockerfile=%[1]s \
-		  --opt filename=%[2]s \
-        --output type=image,name=%[3]s,push=true `, sourcePath, gitSource.Dockerfile, image))
+	// Set common build environment variables
+	addSecret(pCtx, step, "CLOUDNESS_BUILD_SOURCE_PATH", sourcePath)
+	addSecret(pCtx, step, "CLOUDNESS_BUILD_IMAGE", image)
+	addSecret(pCtx, step, "CLOUDNESS_BUILD_CACHE_IMAGE", cacheImage)
+	addSecret(pCtx, step, "CLOUDNESS_IMAGE_REGISTRY", in.Config.PushRegistryURL)
+	addSecret(pCtx, step, "CLOUDNESS_IMAGE_MIRROR_REGISTRY", in.Config.MirrorRegistryURL)
 
-	if cacheImage != "" {
-		cmd.WriteString(fmt.Sprintf(`--export-cache type=registry,ref=%s,mode=max `, cacheImage))
-		cmd.WriteString(fmt.Sprintf(`--import-cache type=registry,ref=%s,mode=max `, cacheImage))
+	if in.Config.MirrorRegistryEnabled && in.Config.MirrorRegistryURL != "" {
+		addVariable(pCtx, step, "CLOUDNESS_MIRROR_ENABLED", "true")
 	}
 
-	// Add BuildArgs
+	// Set build args as space-separated key=value pairs
 	if len(buildVars) > 0 {
+		var args []string
 		for k, v := range buildVars {
-			cmd.WriteString(fmt.Sprintf(`--opt build-arg:%s=%s `, k, v)) //
+			args = append(args, fmt.Sprintf("%s=%s", k, v))
 		}
+		addSecret(pCtx, step, "CLOUDNESS_BUILD_ARGS", strings.Join(args, " "))
 	}
 
-	step.AddScriptCmd(cmd.String())
-}
-
-func updateNixpacksBuildStep(
-	in *pipeline.RunnerContextInput,
-	pCtx *pipeline.RunnerContext,
-	spec *types.ApplicationSpec,
-	step *pipeline.Step,
-	buildVars map[string]string,
-) {
-	gitSource := spec.Build.Source.Git
-	sourcePath := wsBuildVolumePath
-	if gitSource.BasePath != "" && gitSource.BasePath != "/" {
-		sourcePath = wsBuildVolumePath + gitSource.BasePath
-	}
-
-	image, _, cacheImage := specSvc.GetImage(in.Application, in.Deployment, in.Config)
-	nixCommand := []string{fmt.Sprintf("nixpacks build %[1]s -o %[1]s", sourcePath)}
-	nixCommand = append(nixCommand, fmt.Sprintf("--name %s", image))
-	if gitSource.BuildCommand != "" {
-		nixCommand = append(nixCommand, fmt.Sprintf(`--build-cmd "%s"`, gitSource.BuildCommand))
-	}
-	if spec.Deploy.StartCommand != "" {
-		nixCommand = append(nixCommand, fmt.Sprintf(`--start-cmd "%s"`, spec.Deploy.StartCommand))
-	}
-
-	for key, value := range buildVars {
-		nixCommand = append(nixCommand, fmt.Sprintf(`--env %s="%s"`, key, value))
-	}
-	nixCommand = append(nixCommand, "--verbose")
-	step.AddScriptCmd(strings.Join(nixCommand, ` `))
-
-	var cmd strings.Builder
-	//generate buildkit toml
-	addBuildKitConfig(&cmd, in.Config)
-
-	// Construct the buildctl-daemonless.sh command
-	cmd.WriteString(fmt.Sprintf(`buildctl-daemonless.sh build \
-        --frontend=dockerfile.v0 \
-        --local context=%[1]s \
-        --local dockerfile=%[1]s \
-		  --opt filename=/.nixpacks/Dockerfile \
-        --output type=image,name=%[2]s,push=true `, sourcePath, image))
-
-	if cacheImage != "" {
-		cmd.WriteString(fmt.Sprintf(`--export-cache type=registry,ref=%s `, cacheImage))
-		cmd.WriteString(fmt.Sprintf(`--import-cache type=registry,ref=%s,mode=max `, cacheImage))
+	switch specSvc.GetBuilder(spec) {
+	case enum.BuilderTypeDockerfile:
+		addVariable(pCtx, step, "CLOUDNESS_BUILD_TYPE", "dockerfile")
+		addVariable(pCtx, step, "CLOUDNESS_BUILD_DOCKERFILE", gitSource.Dockerfile)
+	case enum.BuilderTypeNixpacks:
+		addSecret(pCtx, step, "CLOUDNESS_BUILD_TYPE", "nixpacks")
+		if gitSource.BuildCommand != "" {
+			addVariable(pCtx, step, "CLOUDNESS_BUILD_CMD", gitSource.BuildCommand)
+		}
+		if spec.Deploy.StartCommand != "" {
+			addVariable(pCtx, step, "CLOUDNESS_START_CMD", spec.Deploy.StartCommand)
+		}
 	}
 
-	step.AddScriptCmd(cmd.String())
-
-}
-
-func addImageSecrets(in *pipeline.RunnerContextInput, pCtx *pipeline.RunnerContext, step *pipeline.Step) {
-	image, pullImage, cacheImage := specSvc.GetImage(in.Application, in.Deployment, in.Config)
-	addSecret(pCtx, step, "CLOUDNESS_BUILD_IMAGE", image)
-	addSecret(pCtx, step, "CLOUDNESS_BUILD_PULL_IMAGE", pullImage)
-	addSecret(pCtx, step, "CLOUDNESS_BUILD_CACHE_IMAGE", cacheImage)
-	addSecret(pCtx, step, "CLOUDNESS_IMAGE_REGISTRY", in.Config.PushRegistryURL)
-	addSecret(pCtx, step, "CLOUDNESS_IMAGE_MIRROR_REGISTRY", in.Config.MirrorRegistryURL)
-}
-
-func addBuildKitConfig(cmd *strings.Builder, config *config.PipelineConfig) {
-	//generate buildkit toml
-	cmd.WriteString(`BUILDKITD_CONFIG_PATH="$HOME/.config/buildkit/buildkitd.toml"` + "\n")
-	cmd.WriteString(`mkdir -p "$(dirname "$BUILDKITD_CONFIG_PATH")"` + "\n")
-	cmd.WriteString(`> "$BUILDKITD_CONFIG_PATH"` + "\n")
-	cmd.WriteString(fmt.Sprintf(`MAIN_REGISTRY=$(echo %s | cut -d'/' -f1)`+"\n", config.PushRegistryURL))
-	cmd.WriteString(`echo "[registry.\"$MAIN_REGISTRY\"]" >> "$BUILDKITD_CONFIG_PATH"` + "\n")
-	cmd.WriteString(`echo "  http = true" >> "$BUILDKITD_CONFIG_PATH"` + "\n")
-	cmd.WriteString(`echo "  insecure = true" >> "$BUILDKITD_CONFIG_PATH"` + "\n")
+	// Add image secrets for other steps
+	addSecrets(pCtx, step, buildVars)
 
-	if config.MirrorRegistryEnabled && config.MirrorRegistryURL != "" {
-		// Add the mirror configuration
-		cmd.WriteString(fmt.Sprintf(`MIRROR_REGISTRY=$(echo %s | cut -d'/' -f1)`+"\n", config.MirrorRegistryURL))
-		cmd.WriteString(`echo "[registry.\"$MIRROR_REGISTRY\"]" >> "$BUILDKITD_CONFIG_PATH"` + "\n")
-		cmd.WriteString(`echo "  http = true" >> "$BUILDKITD_CONFIG_PATH"` + "\n")
-		cmd.WriteString(`echo "  insecure = true" >> "$BUILDKITD_CONFIG_PATH"` + "\n")
+	// Run the build script
+	step.AddScriptCmd(". /usr/local/lib/build-script.sh")
 
-		cmd.WriteString(`echo "[registry.\"docker.io\"]" >> "$BUILDKITD_CONFIG_PATH"` + "\n")
-		cmd.WriteString(fmt.Sprintf(`echo "  mirrors = [\"%s\"]" >> "$BUILDKITD_CONFIG_PATH"`+"\n", config.MirrorRegistryURL))
-	}
+	return nil
 }

app/pipeline/convert/const.go

@@ -7,9 +7,6 @@ import (
 )
 
 const (
-	// images
-	busyBoxImage = "busybox:1.37.0"
-
 	// workspace volume
 	wsBuildVolumePath = "/cloudness/workspace/build"
 
@@ -25,8 +22,8 @@ var (
 func getBuilderImage() string {
 	ver := version.Version
 	if ver.Major == 0 && ver.Minor == 0 && ver.Patch == 0 {
-		return "cloudnessio/builder:latest"
+		return "cloudnessio/helper:latest"
 	}
 	// Use semver String() which includes prerelease (e.g., "0.1.0-alpha.1")
-	return fmt.Sprintf("cloudnessio/builder:v%s", ver.String())
+	return fmt.Sprintf("cloudnessio/helper:v%s", ver.String())
 }

app/pipeline/convert/convert.go

@@ -13,6 +13,7 @@ func ToRunnerContext(in *pipeline.RunnerContextInput) (*pipeline.RunnerContext,
 		Steps:          []*pipeline.Step{},
 		InitSteps:      []*pipeline.Step{},
 		Secrets:        []*pipeline.Secret{},
+		Variables:      []*pipeline.Variable{},
 	}
 
 	pCtx.RunnerName = getRunnerNamespace(pCtx)
@@ -59,7 +60,7 @@ func ToRunnerContext(in *pipeline.RunnerContextInput) (*pipeline.RunnerContext,
 		Envs:           map[string]string{},
 	}
 
-	step.AddStripCmds("#!/bin/sh\n\n", "set -e")
+	step.AddStripCmds("#!/bin/sh\n\n", "set -e", ". /usr/local/lib/cloudness-utils.sh")
 
 	if err := initCommand(step, in, pCtx, spec); err != nil {
 		return nil, err