feat(artifacts): add support for artifacts key

.gitignore

@@ -40,6 +40,7 @@ secrets.env
 
 # Dotenv environment file
 .env
+.env_*
 .env.test
 
 # Files to be excluded.

.golangci.yml

@@ -125,7 +125,8 @@ linters:
     gosec:
       excludes:
         - G101 # hardcoded credentials
-        - G117 # exported field of "Secret" style
+        - G117
+        - G705 # exported field of "Secret" style
     # https://github.com/client9/misspell
     misspell:
       locale: US
@@ -158,6 +159,7 @@ linters:
       allow-first-in-block: true
       allow-whole-block: false
       branch-max-lines: 2
+
   exclusions:
     generated: lax
     presets:

cmd/vela-worker/exec.go

@@ -27,7 +27,10 @@ import (
 //
 //nolint:gocyclo,funlen // ignore cyclomatic complexity and function length
 func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error {
-	var err error
+	var (
+		err       error
+		_executor executor.Engine
+	)
 
 	// setup the version
 	v := version.New()
@@ -144,8 +147,6 @@ func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error
 		// prepare pipeline by hydrating container ID values based on build information
 		p.Prepare(item.Build.GetRepo().GetOrg(), item.Build.GetRepo().GetName(), item.Build.GetNumber(), false)
 
-		logrus.Debugf("setting up exec client with scm token %s with expiration %d", p.Token, p.TokenExp)
-
 		// setup exec client with scm token and build token
 		execBuildClient, err = setupExecClient(w.Config.Server, bt.GetToken(), p.Token, p.TokenExp, item.Build)
 		if err != nil {
@@ -238,11 +239,13 @@ func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error
 	// setup the executor
 	//
 	// https://pkg.go.dev/github.com/go-vela/worker/executor#New
-	_executor, err := executor.New(&executor.Setup{
+	setup := &executor.Setup{
 		Logger:              logger,
 		Mock:                w.Config.Mock,
 		Driver:              w.Config.Executor.Driver,
 		MaxLogSize:          w.Config.Executor.MaxLogSize,
+		FileSizeLimit:       w.Config.Executor.FileSizeLimit,
+		BuildFileSizeLimit:  w.Config.Executor.BuildFileSizeLimit,
 		LogStreamingTimeout: w.Config.Executor.LogStreamingTimeout,
 		EnforceTrustedRepos: w.Config.Executor.EnforceTrustedRepos,
 		PrivilegedImages:    w.Config.Runtime.PrivilegedImages,
@@ -253,8 +256,13 @@ func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error
 		Pipeline:            p.Sanitize(w.Config.Runtime.Driver),
 		Version:             v.Semantic(),
 		OutputCtn:           &execOutputCtn,
-	})
+	}
 
+	_executor, err = executor.New(setup)
+	if err != nil {
+		logger.Errorf("unable to setup executor: %v", err)
+		return err
+	}
 	// add the executor to the worker
 	w.Executors[index] = _executor
 

cmd/vela-worker/flags.go

@@ -61,6 +61,18 @@ func flags() []cli.Flag {
 			Sources: cli.EnvVars("WORKER_BUILD_TIMEOUT", "VELA_BUILD_TIMEOUT", "BUILD_TIMEOUT"),
 			Value:   30 * time.Minute,
 		},
+		&cli.IntFlag{
+			Name:    "storage.file-size-limit",
+			Usage:   "maximum file size (in MB) for a single file upload. 0 means no limit.",
+			Sources: cli.EnvVars("WORKER_STORAGE_FILE_SIZE_LIMIT", "VELA_STORAGE_FILE_SIZE_LIMIT", "STORAGE_FILE_SIZE_LIMIT"),
+			Value:   100,
+		},
+		&cli.IntFlag{
+			Name:    "storage.build-file-size-limit",
+			Usage:   "maximum total size (in MB) for all file uploads in a single build. 0 means no limit.",
+			Sources: cli.EnvVars("WORKER_STORAGE_BUILD_FILE_SIZE_LIMIT", "VELA_STORAGE_BUILD_FILE_SIZE_LIMIT", "STORAGE_BUILD_FILE_SIZE_LIMIT"),
+			Value:   500,
+		},
 
 		// Logger Flags
 

cmd/vela-worker/run.go

@@ -105,6 +105,8 @@ func run(ctx context.Context, c *cli.Command) error {
 			Executor: &executor.Setup{
 				Driver:              c.String("executor.driver"),
 				MaxLogSize:          c.Uint("executor.max_log_size"),
+				FileSizeLimit:       c.Int("storage.file-size-limit"),
+				BuildFileSizeLimit:  c.Int("storage.build-file-size-limit"),
 				LogStreamingTimeout: c.Duration("executor.log_streaming_timeout"),
 				EnforceTrustedRepos: c.Bool("executor.enforce-trusted-repos"),
 				OutputCtn:           outputsCtn,

docker-compose.yml

@@ -83,6 +83,17 @@ services:
       VELA_DISABLE_WEBHOOK_VALIDATION: 'true'
       VELA_ENABLE_SECURE_COOKIE: 'false'
       VELA_REPO_ALLOWLIST: '*'
+      VELA_SCHEDULE_ALLOWLIST: '*'
+      VELA_OTEL_TRACING_ENABLE: true
+      VELA_OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318
+      VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 100
+      VELA_STORAGE_ENABLE: true
+      VELA_STORAGE_DRIVER: minio
+      VELA_STORAGE_ADDRESS: 'http://minio:9001' # Address of the MinIO server
+      VELA_STORAGE_ACCESS_KEY: minioadmin
+      VELA_STORAGE_SECRET_KEY: minioadmin
+      VELA_STORAGE_USE_SSL: 'false'
+      VELA_STORAGE_BUCKET: vela
     env_file:
       - .env
     restart: always
@@ -161,5 +172,42 @@ services:
     cap_add:
       - IPC_LOCK
 
+  # The `minio` compose service hosts the MinIO server instance.
+  #
+  # This component is used for storing build artifacts.
+  #
+  # https://min.io/
+  minio:
+    container_name: minio
+    image: minio/minio
+    restart: always
+    ports:
+      - '9001:9001'
+      - '9002:9002'
+    networks:
+      - vela
+    environment:
+      - MINIO_ROOT_USER=minioadmin
+      - MINIO_ROOT_PASSWORD=minioadmin
+    command: minio server --address ":9001" --console-address ":9002" /data
+
+  # The `minio-setup` compose service is used for setting up the MinIO server instance.
+  #
+  # This component is used for creating the bucket and setting permissions.
+  minio-setup:
+    image: minio/mc
+    container_name: minio-setup
+    depends_on:
+      - minio
+    networks:
+      - vela
+    entrypoint: >
+      /bin/sh -c '
+      until mc alias set local http://minio:9001 minioadmin minioadmin; do sleep 1; done;
+      mc mb --ignore-existing local/vela;
+      mc anonymous set none local/vela;
+      '
+    restart: "no"
+
 networks:
   vela:

executor/linux/build.go

@@ -556,6 +556,13 @@ func (c *client) ExecBuild(ctx context.Context) error {
 		if c.err != nil {
 			return fmt.Errorf("unable to execute step: %w", c.err)
 		}
+
+		if len(_step.Artifacts.Paths) != 0 {
+			err := c.outputs.pollFiles(ctx, c.OutputCtn, _step, c.build)
+			if err != nil {
+				c.Logger.Errorf("unable to poll files for artifacts: %v", err)
+			}
+		}
 	}
 
 	// create an error group with the context for each stage

executor/linux/build_test.go

@@ -190,7 +190,6 @@ func TestLinux_CreateBuild(t *testing.T) {
 				WithBuild(test.build),
 				WithPipeline(_pipeline),
 				WithRuntime(_runtime),
-
 				WithVelaClient(_client),
 			)
 			if err != nil {

executor/linux/driver.go

@@ -8,3 +8,8 @@ import "github.com/go-vela/server/constants"
 func (c *client) Driver() string {
 	return constants.DriverLinux
 }
+
+// StorageDriver outputs the configured storage driver.
+func (c *client) StorageDriver() string {
+	return constants.DriverMinio
+}

executor/linux/linux.go

@@ -29,6 +29,7 @@ type (
 		Hostname     string
 		Version      string
 		OutputCtn    *pipeline.Container
+		Uploaded     int64
 
 		// clients for build actions
 		secret  *secretSvc
@@ -37,6 +38,8 @@ type (
 		// private fields
 		init                *pipeline.Container
 		maxLogSize          uint
+		fileSizeLimit       int64
+		buildFileSizeLimit  int64
 		logStreamingTimeout time.Duration
 		privilegedImages    []string
 		enforceTrustedRepos bool
@@ -74,6 +77,8 @@ func Equal(a, b *client) bool {
 		a.Version == b.Version &&
 		reflect.DeepEqual(a.init, b.init) &&
 		a.maxLogSize == b.maxLogSize &&
+		a.fileSizeLimit == b.fileSizeLimit &&
+		a.buildFileSizeLimit == b.buildFileSizeLimit &&
 		reflect.DeepEqual(a.privilegedImages, b.privilegedImages) &&
 		a.enforceTrustedRepos == b.enforceTrustedRepos &&
 		reflect.DeepEqual(a.build, b.build) &&

executor/linux/opts.go

@@ -47,6 +47,30 @@ func WithMaxLogSize(size uint) Opt {
 	}
 }
 
+// WithFileSizeLimit sets the maximum file size (in MB) for a single file upload in the executor client for Linux.
+func WithFileSizeLimit(limit int) Opt {
+	return func(c *client) error {
+		c.Logger.Trace("configuring file size limit in linux executor client")
+
+		// set the file size limit in the client
+		c.fileSizeLimit = int64(limit) * 1024 * 1024
+
+		return nil
+	}
+}
+
+// WithBuildFileSizeLimit sets the maximum total size (in MB) for all file uploads in a single build in the executor client for Linux.
+func WithBuildFileSizeLimit(limit int) Opt {
+	return func(c *client) error {
+		c.Logger.Trace("configuring build file size limit in linux executor client")
+
+		// set the build file size limit in the client
+		c.buildFileSizeLimit = int64(limit) * 1024 * 1024
+
+		return nil
+	}
+}
+
 // WithLogStreamingTimeout sets the log streaming timeout in the executor client for Linux.
 func WithLogStreamingTimeout(timeout time.Duration) Opt {
 	return func(c *client) error {

executor/linux/opts_test.go

@@ -70,6 +70,53 @@ func TestLinux_Opt_WithBuild(t *testing.T) {
 }
 
 func TestLinux_Opt_WithMaxLogSize(t *testing.T) {
+	// setup tests
+	tests := []struct {
+		name               string
+		failure            bool
+		fileSizeLimit      int
+		buildFileSizeLimit int
+	}{
+		{
+			name:               "defined",
+			failure:            false,
+			fileSizeLimit:      200,
+			buildFileSizeLimit: 1000,
+		},
+	}
+
+	// run tests
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			_engine, err := New(
+				WithFileSizeLimit(test.fileSizeLimit),
+				WithBuildFileSizeLimit(test.buildFileSizeLimit),
+			)
+
+			if test.failure {
+				if err == nil {
+					t.Errorf("WithFileSizeLimit should have returned err")
+				}
+
+				return // continue to next test
+			}
+
+			if err != nil {
+				t.Errorf("WithFileSizeLimit returned err: %v", err)
+			}
+
+			if !reflect.DeepEqual(_engine.fileSizeLimit, int64(test.fileSizeLimit*1024*1024)) {
+				t.Errorf("WithFileSizeLimit is %v, want %v", _engine.fileSizeLimit, int64(test.fileSizeLimit*1024*1024))
+			}
+
+			if !reflect.DeepEqual(_engine.buildFileSizeLimit, int64(test.buildFileSizeLimit*1024*1024)) {
+				t.Errorf("WithBuildFileSizeLimit is %v, want %v", _engine.buildFileSizeLimit, int64(test.buildFileSizeLimit*1024*1024))
+			}
+		})
+	}
+}
+
+func TestLinux_Opt_WithFileSizeLimit(t *testing.T) {
 	// setup tests
 	tests := []struct {
 		name       string