diff --git a/.gitignore b/.gitignore
index 1b32e9a6..5db6e007 100644
--- a/.gitignore
+++ b/.gitignore
@@ -40,6 +40,7 @@ secrets.env
# Dotenv environment file
.env
+.env_*
.env.test
# Files to be excluded.
diff --git a/.golangci.yml b/.golangci.yml
index d6271714..94e5a737 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -125,7 +125,8 @@ linters:
gosec:
excludes:
- G101 # hardcoded credentials
- - G117 # exported field of "Secret" style
+ - G117
+ - G705 # exported field of "Secret" style
# https://github.com/client9/misspell
misspell:
locale: US
@@ -158,6 +159,7 @@ linters:
allow-first-in-block: true
allow-whole-block: false
branch-max-lines: 2
+
exclusions:
generated: lax
presets:
diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go
index ed973fc3..f4f991b7 100644
--- a/cmd/vela-worker/exec.go
+++ b/cmd/vela-worker/exec.go
@@ -27,7 +27,10 @@ import (
//
//nolint:gocyclo,funlen // ignore cyclomatic complexity and function length
func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error {
- var err error
+ var (
+ err error
+ _executor executor.Engine
+ )
// setup the version
v := version.New()
@@ -144,8 +147,6 @@ func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error
// prepare pipeline by hydrating container ID values based on build information
p.Prepare(item.Build.GetRepo().GetOrg(), item.Build.GetRepo().GetName(), item.Build.GetNumber(), false)
- logrus.Debugf("setting up exec client with scm token %s with expiration %d", p.Token, p.TokenExp)
-
// setup exec client with scm token and build token
execBuildClient, err = setupExecClient(w.Config.Server, bt.GetToken(), p.Token, p.TokenExp, item.Build)
if err != nil {
@@ -238,11 +239,13 @@ func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error
// setup the executor
//
// https://pkg.go.dev/github.com/go-vela/worker/executor#New
- _executor, err := executor.New(&executor.Setup{
+ setup := &executor.Setup{
Logger: logger,
Mock: w.Config.Mock,
Driver: w.Config.Executor.Driver,
MaxLogSize: w.Config.Executor.MaxLogSize,
+ FileSizeLimit: w.Config.Executor.FileSizeLimit,
+ BuildFileSizeLimit: w.Config.Executor.BuildFileSizeLimit,
LogStreamingTimeout: w.Config.Executor.LogStreamingTimeout,
EnforceTrustedRepos: w.Config.Executor.EnforceTrustedRepos,
PrivilegedImages: w.Config.Runtime.PrivilegedImages,
@@ -253,8 +256,13 @@ func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error
Pipeline: p.Sanitize(w.Config.Runtime.Driver),
Version: v.Semantic(),
OutputCtn: &execOutputCtn,
- })
+ }
+ _executor, err = executor.New(setup)
+ if err != nil {
+ logger.Errorf("unable to setup executor: %v", err)
+ return err
+ }
// add the executor to the worker
w.Executors[index] = _executor
diff --git a/cmd/vela-worker/flags.go b/cmd/vela-worker/flags.go
index 23fde260..cfa460dc 100644
--- a/cmd/vela-worker/flags.go
+++ b/cmd/vela-worker/flags.go
@@ -61,6 +61,18 @@ func flags() []cli.Flag {
Sources: cli.EnvVars("WORKER_BUILD_TIMEOUT", "VELA_BUILD_TIMEOUT", "BUILD_TIMEOUT"),
Value: 30 * time.Minute,
},
+ &cli.IntFlag{
+ Name: "storage.file-size-limit",
+ Usage: "maximum file size (in MB) for a single file upload. 0 means no limit.",
+ Sources: cli.EnvVars("WORKER_STORAGE_FILE_SIZE_LIMIT", "VELA_STORAGE_FILE_SIZE_LIMIT", "STORAGE_FILE_SIZE_LIMIT"),
+ Value: 100,
+ },
+ &cli.IntFlag{
+ Name: "storage.build-file-size-limit",
+ Usage: "maximum total size (in MB) for all file uploads in a single build. 0 means no limit.",
+ Sources: cli.EnvVars("WORKER_STORAGE_BUILD_FILE_SIZE_LIMIT", "VELA_STORAGE_BUILD_FILE_SIZE_LIMIT", "STORAGE_BUILD_FILE_SIZE_LIMIT"),
+ Value: 500,
+ },
// Logger Flags
diff --git a/cmd/vela-worker/run.go b/cmd/vela-worker/run.go
index 6d1e419d..3d3c2e4e 100644
--- a/cmd/vela-worker/run.go
+++ b/cmd/vela-worker/run.go
@@ -105,6 +105,8 @@ func run(ctx context.Context, c *cli.Command) error {
Executor: &executor.Setup{
Driver: c.String("executor.driver"),
MaxLogSize: c.Uint("executor.max_log_size"),
+ FileSizeLimit: c.Int("storage.file-size-limit"),
+ BuildFileSizeLimit: c.Int("storage.build-file-size-limit"),
LogStreamingTimeout: c.Duration("executor.log_streaming_timeout"),
EnforceTrustedRepos: c.Bool("executor.enforce-trusted-repos"),
OutputCtn: outputsCtn,
diff --git a/docker-compose.yml b/docker-compose.yml
index 0c081b6c..5b464577 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -83,6 +83,17 @@ services:
VELA_DISABLE_WEBHOOK_VALIDATION: 'true'
VELA_ENABLE_SECURE_COOKIE: 'false'
VELA_REPO_ALLOWLIST: '*'
+ VELA_SCHEDULE_ALLOWLIST: '*'
+ VELA_OTEL_TRACING_ENABLE: true
+ VELA_OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318
+ VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 100
+ VELA_STORAGE_ENABLE: true
+ VELA_STORAGE_DRIVER: minio
+ VELA_STORAGE_ADDRESS: 'http://minio:9001' # Address of the MinIO server
+ VELA_STORAGE_ACCESS_KEY: minioadmin
+ VELA_STORAGE_SECRET_KEY: minioadmin
+ VELA_STORAGE_USE_SSL: 'false'
+ VELA_STORAGE_BUCKET: vela
env_file:
- .env
restart: always
@@ -161,5 +172,42 @@ services:
cap_add:
- IPC_LOCK
+ # The `minio` compose service hosts the MinIO server instance.
+ #
+ # This component is used for storing build artifacts.
+ #
+ # https://min.io/
+ minio:
+ container_name: minio
+ image: minio/minio
+ restart: always
+ ports:
+ - '9001:9001'
+ - '9002:9002'
+ networks:
+ - vela
+ environment:
+ - MINIO_ROOT_USER=minioadmin
+ - MINIO_ROOT_PASSWORD=minioadmin
+ command: minio server --address ":9001" --console-address ":9002" /data
+
+ # The `minio-setup` compose service is used for setting up the MinIO server instance.
+ #
+ # This component is used for creating the bucket and setting permissions.
+ minio-setup:
+ image: minio/mc
+ container_name: minio-setup
+ depends_on:
+ - minio
+ networks:
+ - vela
+ entrypoint: >
+ /bin/sh -c '
+ until mc alias set local http://minio:9001 minioadmin minioadmin; do sleep 1; done;
+ mc mb --ignore-existing local/vela;
+ mc anonymous set none local/vela;
+ '
+ restart: "no"
+
networks:
vela:
\ No newline at end of file
diff --git a/executor/linux/build.go b/executor/linux/build.go
index d6f8324c..f697fc77 100644
--- a/executor/linux/build.go
+++ b/executor/linux/build.go
@@ -556,6 +556,13 @@ func (c *client) ExecBuild(ctx context.Context) error {
if c.err != nil {
return fmt.Errorf("unable to execute step: %w", c.err)
}
+
+ if len(_step.Artifacts.Paths) != 0 {
+ err := c.outputs.pollFiles(ctx, c.OutputCtn, _step, c.build)
+ if err != nil {
+ c.Logger.Errorf("unable to poll files for artifacts: %v", err)
+ }
+ }
}
// create an error group with the context for each stage
diff --git a/executor/linux/build_test.go b/executor/linux/build_test.go
index 7d7a8456..35f088de 100644
--- a/executor/linux/build_test.go
+++ b/executor/linux/build_test.go
@@ -190,7 +190,6 @@ func TestLinux_CreateBuild(t *testing.T) {
WithBuild(test.build),
WithPipeline(_pipeline),
WithRuntime(_runtime),
-
WithVelaClient(_client),
)
if err != nil {
diff --git a/executor/linux/driver.go b/executor/linux/driver.go
index 5ef1dd59..ae4821ea 100644
--- a/executor/linux/driver.go
+++ b/executor/linux/driver.go
@@ -8,3 +8,8 @@ import "github.com/go-vela/server/constants"
func (c *client) Driver() string {
return constants.DriverLinux
}
+
+// StorageDriver outputs the configured storage driver.
+func (c *client) StorageDriver() string {
+ return constants.DriverMinio
+}
diff --git a/executor/linux/linux.go b/executor/linux/linux.go
index 83f54dcf..5c85ad3c 100644
--- a/executor/linux/linux.go
+++ b/executor/linux/linux.go
@@ -29,6 +29,7 @@ type (
Hostname string
Version string
OutputCtn *pipeline.Container
+ Uploaded int64
// clients for build actions
secret *secretSvc
@@ -37,6 +38,8 @@ type (
// private fields
init *pipeline.Container
maxLogSize uint
+ fileSizeLimit int64
+ buildFileSizeLimit int64
logStreamingTimeout time.Duration
privilegedImages []string
enforceTrustedRepos bool
@@ -74,6 +77,8 @@ func Equal(a, b *client) bool {
a.Version == b.Version &&
reflect.DeepEqual(a.init, b.init) &&
a.maxLogSize == b.maxLogSize &&
+ a.fileSizeLimit == b.fileSizeLimit &&
+ a.buildFileSizeLimit == b.buildFileSizeLimit &&
reflect.DeepEqual(a.privilegedImages, b.privilegedImages) &&
a.enforceTrustedRepos == b.enforceTrustedRepos &&
reflect.DeepEqual(a.build, b.build) &&
diff --git a/executor/linux/opts.go b/executor/linux/opts.go
index 5c394bae..469870d5 100644
--- a/executor/linux/opts.go
+++ b/executor/linux/opts.go
@@ -47,6 +47,30 @@ func WithMaxLogSize(size uint) Opt {
}
}
+// WithFileSizeLimit sets the maximum file size (in MB) for a single file upload in the executor client for Linux.
+func WithFileSizeLimit(limit int) Opt {
+ return func(c *client) error {
+ c.Logger.Trace("configuring file size limit in linux executor client")
+
+ // set the file size limit in the client
+ c.fileSizeLimit = int64(limit) * 1024 * 1024
+
+ return nil
+ }
+}
+
+// WithBuildFileSizeLimit sets the maximum total size (in MB) for all file uploads in a single build in the executor client for Linux.
+func WithBuildFileSizeLimit(limit int) Opt {
+ return func(c *client) error {
+ c.Logger.Trace("configuring build file size limit in linux executor client")
+
+ // set the build file size limit in the client
+ c.buildFileSizeLimit = int64(limit) * 1024 * 1024
+
+ return nil
+ }
+}
+
// WithLogStreamingTimeout sets the log streaming timeout in the executor client for Linux.
func WithLogStreamingTimeout(timeout time.Duration) Opt {
return func(c *client) error {
diff --git a/executor/linux/opts_test.go b/executor/linux/opts_test.go
index 36196330..2f3f1a1a 100644
--- a/executor/linux/opts_test.go
+++ b/executor/linux/opts_test.go
@@ -70,6 +70,53 @@ func TestLinux_Opt_WithBuild(t *testing.T) {
}
func TestLinux_Opt_WithMaxLogSize(t *testing.T) {
+ // setup tests
+ tests := []struct {
+ name string
+ failure bool
+ fileSizeLimit int
+ buildFileSizeLimit int
+ }{
+ {
+ name: "defined",
+ failure: false,
+ fileSizeLimit: 200,
+ buildFileSizeLimit: 1000,
+ },
+ }
+
+ // run tests
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ _engine, err := New(
+ WithFileSizeLimit(test.fileSizeLimit),
+ WithBuildFileSizeLimit(test.buildFileSizeLimit),
+ )
+
+ if test.failure {
+ if err == nil {
+ t.Errorf("WithFileSizeLimit should have returned err")
+ }
+
+ return // continue to next test
+ }
+
+ if err != nil {
+ t.Errorf("WithFileSizeLimit returned err: %v", err)
+ }
+
+ if !reflect.DeepEqual(_engine.fileSizeLimit, int64(test.fileSizeLimit*1024*1024)) {
+ t.Errorf("WithFileSizeLimit is %v, want %v", _engine.fileSizeLimit, int64(test.fileSizeLimit*1024*1024))
+ }
+
+ if !reflect.DeepEqual(_engine.buildFileSizeLimit, int64(test.buildFileSizeLimit*1024*1024)) {
+ t.Errorf("WithBuildFileSizeLimit is %v, want %v", _engine.buildFileSizeLimit, int64(test.buildFileSizeLimit*1024*1024))
+ }
+ })
+ }
+}
+
+func TestLinux_Opt_WithFileSizeLimit(t *testing.T) {
// setup tests
tests := []struct {
name string
diff --git a/executor/linux/outputs.go b/executor/linux/outputs.go
index b622a859..cf34f632 100644
--- a/executor/linux/outputs.go
+++ b/executor/linux/outputs.go
@@ -8,11 +8,15 @@ import (
"encoding/base64"
"fmt"
"maps"
+ "path/filepath"
+ "strconv"
envparse "github.com/hashicorp/go-envparse"
"github.com/sirupsen/logrus"
+ api "github.com/go-vela/server/api/types"
"github.com/go-vela/server/compiler/types/pipeline"
+ "github.com/go-vela/server/storage"
)
// outputSvc handles communication with the outputs container during the build.
@@ -204,3 +208,97 @@ func (o *outputSvc) poll(ctx context.Context, ctn *pipeline.Container) (map[stri
return outputMap, maskMap, nil
}
+
+// pollFiles polls the output for files from the sidecar container.
+func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, _step *pipeline.Container, b *api.Build) error {
+ // exit if outputs container has not been configured
+ if len(ctn.Image) == 0 {
+ return fmt.Errorf("no outputs container configured")
+ }
+
+ // update engine logger with outputs metadata
+ //
+ // https://pkg.go.dev/github.com/sirupsen/logrus#Entry.WithField
+ logger := o.client.Logger.WithField("artifact-outputs", ctn.Name)
+
+ creds, res, err := o.client.Vela.Build.GetSTSCreds(ctx, b.GetRepo().GetOrg(), b.GetRepo().GetName(),
+ b.GetNumber())
+ if err != nil {
+ return fmt.Errorf("unable to get sts storage creds %w with response code %d", err, res.StatusCode)
+ }
+
+ stsStorageClient, err := storage.New(&storage.Setup{
+ Enable: creds.Enable,
+ Driver: creds.Driver,
+ Endpoint: creds.Endpoint,
+ AccessKey: creds.AccessKey,
+ SecretKey: creds.SecretKey,
+ Bucket: creds.Bucket,
+ Secure: creds.Secure,
+ Token: creds.SessionToken,
+ })
+ if err != nil {
+ return fmt.Errorf("unable to create sts storage client %w with response code %d", err, res.StatusCode)
+ }
+
+ if stsStorageClient == nil {
+ return fmt.Errorf("sts storage client is nil")
+ }
+
+ // grab file paths from the container
+ filesPath, err := o.client.Runtime.PollFileNames(ctx, ctn, _step)
+ if err != nil {
+ return fmt.Errorf("unable to poll file names: %w", err)
+ }
+
+ if len(filesPath) == 0 {
+ return fmt.Errorf("no files found for file list: %v", _step.Artifacts.Paths)
+ }
+
+ // process each file found
+ for _, filePath := range filesPath {
+ fileName := filepath.Base(filePath)
+ logger.Debugf("processing file: %s (path: %s)", fileName, filePath)
+
+ // get file content from container
+ reader, size, err := o.client.Runtime.PollFileContent(ctx, ctn, filePath)
+ if err != nil {
+ return fmt.Errorf("unable to poll file content for %s: %w", filePath, err)
+ }
+
+ // TODO: surface this skip to the user
+ if o.client.fileSizeLimit > 0 && size > o.client.fileSizeLimit {
+ logger.Infof("skipping file %s due to file size limit", filePath)
+ continue
+ }
+
+ if o.client.buildFileSizeLimit > 0 && size+o.client.Uploaded > o.client.buildFileSizeLimit {
+ logger.Infof("skipping file %s due to build file size limit", filePath)
+ continue
+ }
+
+ // create storage object path
+ objectName := fmt.Sprintf("%s/%s/%s/%s",
+ b.GetRepo().GetOrg(),
+ b.GetRepo().GetName(),
+ strconv.FormatInt(b.GetNumber(), 10),
+ fileName)
+
+ obj := &api.Object{
+ Bucket: api.Bucket{
+ BucketName: creds.Bucket,
+ },
+ ObjectName: objectName,
+ FilePath: filePath,
+ }
+ // upload file to storage
+ err = stsStorageClient.UploadObject(ctx, obj, reader, size)
+ if err != nil {
+ return fmt.Errorf("unable to upload object %s: %w", fileName, err)
+ }
+
+ o.client.Uploaded += size
+ }
+
+ return nil
+}
diff --git a/executor/linux/stage.go b/executor/linux/stage.go
index 41399b37..fcb99236 100644
--- a/executor/linux/stage.go
+++ b/executor/linux/stage.go
@@ -198,6 +198,13 @@ func (c *client) ExecStage(ctx context.Context, s *pipeline.Stage, m *sync.Map)
if _step.ExitCode != 0 && !_step.Ruleset.Continue {
stageStatus = constants.StatusFailure
}
+
+ if len(_step.Artifacts.Paths) != 0 {
+ err := c.outputs.pollFiles(ctx, c.OutputCtn, _step, c.build)
+ if err != nil {
+ c.Logger.Errorf("unable to poll files for artifacts: %v", err)
+ }
+ }
}
return nil
diff --git a/executor/local/service.go b/executor/local/service.go
index 34186a03..aded9ace 100644
--- a/executor/local/service.go
+++ b/executor/local/service.go
@@ -119,8 +119,6 @@ func (c *client) StreamService(ctx context.Context, ctn *pipeline.Container) err
// scan entire container output
for scanner.Scan() {
// ensure we output to stdout
- //
- //nolint:gosec // false positive
fmt.Fprintln(c.stdout, _pattern, scanner.Text())
}
diff --git a/executor/local/step.go b/executor/local/step.go
index dc70b054..ac12a79d 100644
--- a/executor/local/step.go
+++ b/executor/local/step.go
@@ -195,8 +195,6 @@ func (c *client) StreamStep(ctx context.Context, ctn *pipeline.Container) error
// scan entire container output
for scanner.Scan() {
// ensure we output to stdout
- //
- //nolint:gosec // false positive
fmt.Fprintln(c.stdout, _pattern, scanner.Text())
}
diff --git a/executor/setup.go b/executor/setup.go
index bd2ada04..a776e4ea 100644
--- a/executor/setup.go
+++ b/executor/setup.go
@@ -34,6 +34,10 @@ type Setup struct {
Driver string
// specifies the maximum log size
MaxLogSize uint
+ // specifies file size limit for artifacts
+ FileSizeLimit int
+ // specifies build file size limit for artifacts
+ BuildFileSizeLimit int
// specifies how long to wait after the build finishes
// for log streaming to complete
LogStreamingTimeout time.Duration
@@ -75,12 +79,12 @@ func (s *Setup) Darwin() (Engine, error) {
func (s *Setup) Linux() (Engine, error) {
logrus.Trace("creating linux executor client from setup")
- // create new Linux executor engine
- //
- // https://pkg.go.dev/github.com/go-vela/worker/executor/linux#New
- return linux.New(
+ // create options for Linux executor
+ opts := []linux.Opt{
linux.WithBuild(s.Build),
linux.WithMaxLogSize(s.MaxLogSize),
+ linux.WithFileSizeLimit(s.FileSizeLimit),
+ linux.WithBuildFileSizeLimit(s.BuildFileSizeLimit),
linux.WithLogStreamingTimeout(s.LogStreamingTimeout),
linux.WithPrivilegedImages(s.PrivilegedImages),
linux.WithEnforceTrustedRepos(s.EnforceTrustedRepos),
@@ -91,7 +95,11 @@ func (s *Setup) Linux() (Engine, error) {
linux.WithVersion(s.Version),
linux.WithLogger(s.Logger),
linux.WithOutputCtn(s.OutputCtn),
- )
+ }
+ // create new Linux executor engine
+ //
+ // https://pkg.go.dev/github.com/go-vela/worker/executor/linux#New
+ return linux.New(opts...)
}
// Local creates and returns a Vela engine capable of
@@ -99,10 +107,7 @@ func (s *Setup) Linux() (Engine, error) {
func (s *Setup) Local() (Engine, error) {
logrus.Trace("creating local executor client from setup")
- // create new Local executor engine
- //
- // https://pkg.go.dev/github.com/go-vela/worker/executor/local#New
- return local.New(
+ opts := []local.Opt{
local.WithBuild(s.Build),
local.WithHostname(s.Hostname),
local.WithPipeline(s.Pipeline),
@@ -111,7 +116,12 @@ func (s *Setup) Local() (Engine, error) {
local.WithVersion(s.Version),
local.WithMockStdout(s.Mock),
local.WithOutputCtn(s.OutputCtn),
- )
+ }
+
+ // create new Local executor engine
+ //
+ // https://pkg.go.dev/github.com/go-vela/worker/executor/local#New
+ return local.New(opts...)
}
// Windows creates and returns a Vela engine capable of
diff --git a/executor/setup_test.go b/executor/setup_test.go
index e89700a7..c48e7f26 100644
--- a/executor/setup_test.go
+++ b/executor/setup_test.go
@@ -72,6 +72,8 @@ func TestExecutor_Setup_Linux(t *testing.T) {
want, err := linux.New(
linux.WithBuild(_build),
linux.WithMaxLogSize(2097152),
+ linux.WithFileSizeLimit(10),
+ linux.WithBuildFileSizeLimit(100),
linux.WithLogStreamingTimeout(1*time.Second),
linux.WithHostname("localhost"),
linux.WithPipeline(_pipeline),
@@ -84,14 +86,16 @@ func TestExecutor_Setup_Linux(t *testing.T) {
}
_setup := &Setup{
- Build: _build,
- Client: _client,
- Driver: constants.DriverLinux,
- MaxLogSize: 2097152,
- Hostname: "localhost",
- Pipeline: _pipeline,
- Runtime: _runtime,
- Version: "v1.0.0",
+ Build: _build,
+ Client: _client,
+ Driver: constants.DriverLinux,
+ MaxLogSize: 2097152,
+ FileSizeLimit: 10,
+ BuildFileSizeLimit: 100,
+ Hostname: "localhost",
+ Pipeline: _pipeline,
+ Runtime: _runtime,
+ Version: "v1.0.0",
}
// run test
diff --git a/mock/docker/container.go b/mock/docker/container.go
index 419ab638..2f2ac60a 100644
--- a/mock/docker/container.go
+++ b/mock/docker/container.go
@@ -4,12 +4,14 @@ package docker
import (
"archive/tar"
+ "bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
+ "net"
"strings"
"time"
@@ -24,6 +26,19 @@ import (
v1 "github.com/opencontainers/image-spec/specs-go/v1"
)
+// mockConn is a mock implementation of net.Conn for testing purposes.
+// It provides no-op implementations of all net.Conn methods.
+type mockConn struct{}
+
+func (m *mockConn) Read(b []byte) (n int, err error) { return 0, io.EOF }
+func (m *mockConn) Write(b []byte) (n int, err error) { return len(b), nil }
+func (m *mockConn) Close() error { return nil }
+func (m *mockConn) LocalAddr() net.Addr { return nil }
+func (m *mockConn) RemoteAddr() net.Addr { return nil }
+func (m *mockConn) SetDeadline(t time.Time) error { return nil }
+func (m *mockConn) SetReadDeadline(t time.Time) error { return nil }
+func (m *mockConn) SetWriteDeadline(t time.Time) error { return nil }
+
// ContainerService implements all the container
// related functions for the Docker mock.
type ContainerService struct{}
@@ -106,8 +121,51 @@ func (c *ContainerService) ContainerDiff(_ context.Context, _ string) ([]contain
// running inside a Docker container.
//
// https://pkg.go.dev/github.com/docker/docker/client#Client.ContainerExecAttach
-func (c *ContainerService) ContainerExecAttach(_ context.Context, _ string, _ container.ExecAttachOptions) (types.HijackedResponse, error) {
- return types.HijackedResponse{}, nil
+func (c *ContainerService) ContainerExecAttach(_ context.Context, execID string, _ container.ExecAttachOptions) (types.HijackedResponse, error) {
+ // create a buffer to hold mock output
+ var buf bytes.Buffer
+
+ // write mock stdout using stdcopy format
+ stdoutWriter := stdcopy.NewStdWriter(&buf, stdcopy.Stdout)
+ stderrWriter := stdcopy.NewStdWriter(&buf, stdcopy.Stderr)
+
+ // check for specific test scenarios based on execID
+ if strings.Contains(execID, "error") {
+ // write to stderr to simulate an error
+ _, _ = stderrWriter.Write([]byte("mock exec error"))
+ } else if strings.Contains(execID, "multiline") {
+ // write multiple lines for testing
+ _, _ = stdoutWriter.Write([]byte("line1\nline2\nline3"))
+ } else if strings.Contains(execID, "artifacts-find") {
+ // simulate find command for artifacts - return file paths
+ _, _ = stdoutWriter.Write([]byte("/vela/artifacts/test_results/alpha.txt\n/vela/artifacts/test_results/beta.txt\n/vela/artifacts/build_results/alpha.txt\n/vela/artifacts/build_results/beta.txt"))
+ } else if strings.Contains(execID, "test-results-xml") {
+ // simulate find command for test results XML files
+ _, _ = stdoutWriter.Write([]byte("/vela/workspace/test-results/junit.xml\n/vela/workspace/test-results/report.xml"))
+ } else if strings.Contains(execID, "cypress-screenshots") {
+ // simulate find command for cypress screenshots
+ _, _ = stdoutWriter.Write([]byte("/vela/workspace/cypress/screenshots/test1/screenshot1.png\n/vela/workspace/cypress/screenshots/test1/screenshot2.png\n/vela/workspace/cypress/screenshots/test2/error.png"))
+ } else if strings.Contains(execID, "cypress-videos") {
+ // simulate find command for cypress videos
+ _, _ = stdoutWriter.Write([]byte("/vela/workspace/cypress/videos/test1.mp4\n/vela/workspace/cypress/videos/test2.mp4"))
+ } else if strings.Contains(execID, "cypress-all") {
+ // simulate find command for all cypress artifacts (screenshots + videos)
+ _, _ = stdoutWriter.Write([]byte("/vela/workspace/cypress/screenshots/test1/screenshot1.png\n/vela/workspace/cypress/screenshots/test2/error.png\n/vela/workspace/cypress/videos/test1.mp4\n/vela/workspace/cypress/videos/test2.mp4"))
+ } else if strings.Contains(execID, "not-found") {
+ // simulate path not found
+ _, _ = stderrWriter.Write([]byte("find: '/not-found': No such file or directory"))
+ } else {
+ // default mock output
+ _, _ = stdoutWriter.Write([]byte("mock exec output"))
+ }
+
+ // create a HijackedResponse with the mock data
+ response := types.HijackedResponse{
+ Reader: bufio.NewReader(&buf),
+ Conn: &mockConn{}, // Use mock connection to avoid nil pointer dereference
+ }
+
+ return response, nil
}
// ContainerExecCreate is a helper function to simulate
@@ -115,8 +173,52 @@ func (c *ContainerService) ContainerExecAttach(_ context.Context, _ string, _ co
// Docker container.
//
// https://pkg.go.dev/github.com/docker/docker/client#Client.ContainerExecCreate
-func (c *ContainerService) ContainerExecCreate(_ context.Context, _ string, _ container.ExecOptions) (container.ExecCreateResponse, error) {
- return container.ExecCreateResponse{}, nil
+func (c *ContainerService) ContainerExecCreate(_ context.Context, ctn string, config container.ExecOptions) (container.ExecCreateResponse, error) {
+ // verify a container was provided
+ if len(ctn) == 0 {
+ return container.ExecCreateResponse{}, errors.New("no container provided")
+ }
+
+ // check if the container is not found
+ if strings.Contains(ctn, "notfound") || strings.Contains(ctn, "not-found") {
+ return container.ExecCreateResponse{}, errdefs.NotFound(fmt.Errorf("Error: No such container: %s", ctn))
+ }
+
+ // create exec ID based on command for testing scenarios
+ execID := stringid.GenerateRandomID()
+
+ // check command for specific test scenarios
+ if len(config.Cmd) > 0 {
+ cmdStr := strings.Join(config.Cmd, " ")
+ if strings.Contains(cmdStr, "error") {
+ execID = "exec-error-" + execID
+ } else if strings.Contains(cmdStr, "multiline") {
+ execID = "exec-multiline-" + execID
+ } else if strings.Contains(cmdStr, "find") {
+ // For artifact file search commands
+ if strings.Contains(cmdStr, "/not-found") {
+ execID = "exec-not-found-" + execID
+ } else if strings.Contains(cmdStr, "artifacts") {
+ execID = "exec-artifacts-find-" + execID
+ } else if strings.Contains(cmdStr, "test-results") && strings.Contains(cmdStr, ".xml") {
+ execID = "exec-test-results-xml-" + execID
+ } else if strings.Contains(cmdStr, "cypress/screenshots") && strings.Contains(cmdStr, ".png") {
+ execID = "exec-cypress-screenshots-" + execID
+ } else if strings.Contains(cmdStr, "cypress/videos") && strings.Contains(cmdStr, ".mp4") {
+ execID = "exec-cypress-videos-" + execID
+ } else if strings.Contains(cmdStr, "cypress") {
+ // Generic cypress pattern for combined searches
+ execID = "exec-cypress-all-" + execID
+ }
+ }
+ }
+
+ // create response object to return
+ response := container.ExecCreateResponse{
+ ID: execID,
+ }
+
+ return response, nil
}
// ContainerExecInspect is a helper function to simulate
@@ -523,7 +625,19 @@ func (c *ContainerService) CopyFromContainer(_ context.Context, _ string, path s
tw := tar.NewWriter(&buf)
- content := []byte("key=value")
+ // Determine content based on path for test scenarios
+ var content []byte
+ if strings.Contains(path, "artifacts") && strings.Contains(path, "alpha.txt") {
+ content = []byte("results")
+ } else if strings.Contains(path, "test-results") && strings.Contains(path, ".xml") {
+ content = []byte("")
+ } else if strings.Contains(path, "cypress/screenshots") && strings.Contains(path, ".png") {
+ content = []byte("PNG_BINARY_DATA")
+ } else if strings.Contains(path, "cypress/videos") && strings.Contains(path, ".mp4") {
+ content = []byte("MP4_BINARY_DATA")
+ } else {
+ content = []byte("key=value")
+ }
hdr := &tar.Header{
Name: path,
diff --git a/runtime/docker/artifact.go b/runtime/docker/artifact.go
new file mode 100644
index 00000000..cccbead4
--- /dev/null
+++ b/runtime/docker/artifact.go
@@ -0,0 +1,163 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package docker
+
+import (
+ "archive/tar"
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "path/filepath"
+ "strings"
+
+ "github.com/containerd/errdefs"
+ dockerContainerTypes "github.com/docker/docker/api/types/container"
+ "github.com/docker/docker/pkg/stdcopy"
+
+ "github.com/go-vela/server/compiler/types/pipeline"
+)
+
+// execContainerLines runs `sh -c cmd` in the named container and
+// returns its stdout split by newline (error if anything on stderr).
+func (c *client) execContainerLines(ctx context.Context, containerID, cmd string) ([]string, error) {
+ execConfig := dockerContainerTypes.ExecOptions{
+ Tty: true,
+ Cmd: []string{"sh", "-c", cmd},
+ AttachStdout: true,
+ AttachStderr: true,
+ }
+
+ resp, err := c.Docker.ContainerExecCreate(ctx, containerID, execConfig)
+ if err != nil {
+ return nil, fmt.Errorf("create exec: %w", err)
+ }
+
+ attach, err := c.Docker.ContainerExecAttach(ctx, resp.ID, dockerContainerTypes.ExecAttachOptions{})
+ if err != nil {
+ return nil, fmt.Errorf("attach exec: %w", err)
+ }
+
+ defer attach.Close()
+
+ var outBuf, errBuf bytes.Buffer
+ if _, err := stdcopy.StdCopy(&outBuf, &errBuf, attach.Reader); err != nil {
+ return nil, fmt.Errorf("copy exec output: %w", err)
+ }
+
+ if errBuf.Len() > 0 {
+ return nil, fmt.Errorf("exec error: %s", errBuf.String())
+ }
+
+ lines := strings.Split(strings.TrimSpace(outBuf.String()), "\n")
+
+ return lines, nil
+}
+
+// PollFileNames searches for files matching the provided patterns within a container.
+func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, _step *pipeline.Container) ([]string, error) {
+ c.Logger.Tracef("gathering files from container %s", ctn.ID)
+
+ if ctn.Image == "" {
+ return nil, nil
+ }
+
+ var results []string
+
+ paths := _step.Artifacts.Paths
+
+ for _, pattern := range paths {
+ // use find command to locate files matching the pattern
+ cmd := fmt.Sprintf("find %s -type f -path '*%s' -print", _step.Environment["VELA_WORKSPACE"], pattern)
+ c.Logger.Debugf("searching for files with pattern: %s", pattern)
+
+ lines, err := c.execContainerLines(ctx, ctn.ID, cmd)
+ if err != nil {
+ return nil, fmt.Errorf("failed to search for pattern %q: %w", pattern, err)
+ }
+
+ c.Logger.Tracef("found %d candidates for pattern %s", len(lines), pattern)
+
+ // process each found file
+ for _, line := range lines {
+ filePath := filepath.Clean(strings.TrimSpace(line))
+ if filePath == "" {
+ continue
+ }
+
+ c.Logger.Debugf("accepted file: %s", filePath)
+ results = append(results, filePath)
+ }
+ }
+
+ if len(results) == 0 {
+ return results, fmt.Errorf("no matching files found for patterns: %v", paths)
+ }
+
+ return results, nil
+}
+
+// PollFileContent retrieves the content and size of a file inside a container.
+func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, path string) (io.Reader, int64, error) {
+ c.Logger.Tracef("gathering test results and attachments from container %s", ctn.ID)
+
+ if len(ctn.Image) == 0 || len(path) == 0 {
+ return nil, 0, nil
+ }
+
+ // copy file from outputs container
+ reader, _, err := c.Docker.CopyFromContainer(ctx, ctn.ID, path)
+ if err != nil {
+ c.Logger.Debugf("PollFileContent CopyFromContainer failed for %q: %v", path, err)
+ // early non-error exit if not found
+ if errdefs.IsNotFound(err) {
+ return nil, 0, nil
+ }
+
+ return nil, 0, err
+ }
+
+ defer reader.Close()
+
+ // docker returns a tar archive for the path
+ tr := tar.NewReader(reader)
+
+ header, err := tr.Next()
+ if err != nil {
+ // if the tar has no entries or is finished unexpectedly
+ if errors.Is(err, io.EOF) {
+ c.Logger.Debugf("PollFileContent: no tar entries for %q", path)
+
+ return nil, 0, nil
+ }
+
+ c.Logger.Debugf("PollFileContent tr.Next failed for %q: %v", path, err)
+
+ return nil, 0, err
+ }
+
+ // Ensure the tar entry is a regular file (not dir, symlink, etc.)
+ if header.Typeflag != tar.TypeReg {
+ c.Logger.Debugf("PollFileContent unexpected tar entry type %v for %q", header.Typeflag, path)
+
+ return nil, 0, fmt.Errorf("unexpected tar entry type %v for %q", header.Typeflag, path)
+ }
+
+ // Read file contents. Use io.ReadAll to avoid dealing with CopyN EOF nuances.
+ fileBytes, err := io.ReadAll(tr)
+ if err != nil {
+ c.Logger.Debugf("PollFileContent ReadAll failed for %q: %v", path, err)
+
+ return nil, 0, err
+ }
+
+ if len(fileBytes) == 0 {
+ c.Logger.Errorf("PollFileContent returned no data for path: %s", path)
+
+ return nil, 0, fmt.Errorf("no data returned from container for %q", path)
+ }
+
+ // Return a reader and length (use int64 for size)
+ return bytes.NewReader(fileBytes), int64(len(fileBytes)), nil
+}
diff --git a/runtime/docker/artifact_test.go b/runtime/docker/artifact_test.go
new file mode 100644
index 00000000..48887f6a
--- /dev/null
+++ b/runtime/docker/artifact_test.go
@@ -0,0 +1,309 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package docker
+
+import (
+ "context"
+ "io"
+ "testing"
+
+ "github.com/go-vela/server/compiler/types/pipeline"
+)
+
+func TestDocker_PollFileNames(t *testing.T) {
+ // setup Docker
+ _engine, err := NewMock()
+ if err != nil {
+ t.Errorf("unable to create runtime engine: %v", err)
+ }
+
+ // setup tests
+ tests := []struct {
+ name string
+ failure bool
+ container *pipeline.Container
+ step *pipeline.Container
+ wantFiles int
+ }{
+ {
+ name: "artifacts directory search",
+ failure: false,
+ container: &pipeline.Container{
+ ID: "artifacts-container",
+ Image: "alpine:latest",
+ },
+ step: &pipeline.Container{
+ Artifacts: pipeline.Artifacts{
+ Paths: []string{"artifacts/*/*.txt"},
+ },
+ Environment: map[string]string{
+ "VELA_WORKSPACE": "/vela/workspace",
+ },
+ },
+ wantFiles: 4, // alpha.txt and beta.txt in test_results and build_results
+ },
+ {
+ name: "test-results XML files",
+ failure: false,
+ container: &pipeline.Container{
+ ID: "test-results-container",
+ Image: "alpine:latest",
+ },
+ step: &pipeline.Container{
+ Artifacts: pipeline.Artifacts{
+ Paths: []string{"test-results/*.xml"},
+ },
+ Environment: map[string]string{
+ "VELA_WORKSPACE": "/vela/workspace",
+ },
+ },
+ wantFiles: 2, // junit.xml and report.xml
+ },
+ {
+ name: "cypress screenshots PNG files",
+ failure: false,
+ container: &pipeline.Container{
+ ID: "cypress-screenshots-container",
+ Image: "cypress/browsers:latest",
+ },
+ step: &pipeline.Container{
+ Artifacts: pipeline.Artifacts{
+ Paths: []string{"cypress/screenshots/**/*.png"},
+ },
+ Environment: map[string]string{
+ "VELA_WORKSPACE": "/vela/workspace",
+ },
+ },
+ wantFiles: 3, // screenshot1.png, screenshot2.png, error.png
+ },
+ {
+ name: "cypress videos MP4 files",
+ failure: false,
+ container: &pipeline.Container{
+ ID: "cypress-videos-container",
+ Image: "cypress/browsers:latest",
+ },
+ step: &pipeline.Container{
+ Artifacts: pipeline.Artifacts{
+ Paths: []string{"cypress/videos/**/*.mp4"},
+ },
+ Environment: map[string]string{
+ "VELA_WORKSPACE": "/vela/workspace",
+ },
+ },
+ wantFiles: 2, // test1.mp4, test2.mp4
+ },
+ {
+ name: "multiple cypress patterns",
+ failure: false,
+ container: &pipeline.Container{
+ ID: "cypress-all-container",
+ Image: "cypress/browsers:latest",
+ },
+ step: &pipeline.Container{
+ Artifacts: pipeline.Artifacts{
+ Paths: []string{
+ "cypress/screenshots/**/*.png",
+ "cypress/videos/**/*.mp4",
+ },
+ },
+ Environment: map[string]string{
+ "VELA_WORKSPACE": "/vela/workspace",
+ },
+ },
+ wantFiles: 5, // 3 PNG screenshots + 2 MP4 videos
+ },
+ {
+ name: "combined test-results and cypress artifacts",
+ failure: false,
+ container: &pipeline.Container{
+ ID: "combined-artifacts-container",
+ Image: "cypress/browsers:latest",
+ },
+ step: &pipeline.Container{
+ Artifacts: pipeline.Artifacts{
+ Paths: []string{
+ "test-results/*.xml",
+ "cypress/screenshots/**/*.png",
+ "cypress/videos/**/*.mp4",
+ },
+ },
+ Environment: map[string]string{
+ "VELA_WORKSPACE": "/vela/workspace",
+ },
+ },
+ wantFiles: 7, // 2 XML + 3 PNG + 2 MP4
+ },
+ {
+ name: "directory not found",
+ failure: true,
+ container: &pipeline.Container{
+ ID: "artifacts-container",
+ Image: "alpine:latest",
+ },
+ step: &pipeline.Container{
+ Artifacts: pipeline.Artifacts{
+ Paths: []string{"artifacts/*.txt"},
+ },
+ Environment: map[string]string{
+ "VELA_WORKSPACE": "/not-found",
+ },
+ },
+ },
+ {
+ name: "empty container image",
+ failure: false,
+ container: &pipeline.Container{
+ ID: "no-image",
+ Image: "",
+ },
+ step: &pipeline.Container{
+ Artifacts: pipeline.Artifacts{
+ Paths: []string{"*.txt"},
+ },
+ Environment: map[string]string{
+ "VELA_WORKSPACE": "/vela/workspace",
+ },
+ },
+ },
+ }
+
+ // run tests
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ got, err := _engine.PollFileNames(context.Background(), test.container, test.step)
+
+ if test.failure {
+ if err == nil {
+ t.Errorf("PollFileNames should have returned err")
+ }
+
+ return // continue to next test
+ }
+
+ if err != nil {
+ t.Errorf("PollFileNames returned err: %v", err)
+ }
+
+ if test.wantFiles > 0 && len(got) != test.wantFiles {
+ t.Errorf("PollFileNames returned %d files, want %d", len(got), test.wantFiles)
+ }
+ })
+ }
+}
+
+func TestDocker_PollFileContent(t *testing.T) {
+ // setup Docker
+ _engine, err := NewMock()
+ if err != nil {
+ t.Errorf("unable to create runtime engine: %v", err)
+ }
+
+ // setup tests
+ tests := []struct {
+ name string
+ failure bool
+ container *pipeline.Container
+ path string
+ wantBytes []byte
+ }{
+ {
+ name: "file content from default path",
+ failure: false,
+ container: &pipeline.Container{
+ ID: "content-container",
+ Image: "alpine:latest",
+ },
+ path: "/vela/artifacts/test_results/alpha.txt",
+ wantBytes: []byte("results"),
+ },
+ {
+ name: "test-results XML file content",
+ failure: false,
+ container: &pipeline.Container{
+ ID: "content-container",
+ Image: "alpine:latest",
+ },
+ path: "/vela/workspace/test-results/junit.xml",
+ wantBytes: []byte(""),
+ },
+ {
+ name: "cypress screenshot PNG file",
+ failure: false,
+ container: &pipeline.Container{
+ ID: "content-container",
+ Image: "cypress/browsers:latest",
+ },
+ path: "/vela/workspace/cypress/screenshots/test1/screenshot1.png",
+ wantBytes: []byte("PNG_BINARY_DATA"),
+ },
+ {
+ name: "cypress video MP4 file",
+ failure: false,
+ container: &pipeline.Container{
+ ID: "content-container",
+ Image: "cypress/browsers:latest",
+ },
+ path: "/vela/workspace/cypress/videos/test1.mp4",
+ wantBytes: []byte("MP4_BINARY_DATA"),
+ },
+ {
+ name: "path not found",
+ failure: false,
+ container: &pipeline.Container{
+ ID: "content-container",
+ Image: "alpine:latest",
+ },
+ path: "not-found",
+ },
+ {
+ name: "empty container image",
+ failure: false,
+ container: new(pipeline.Container),
+ path: "/some/path",
+ },
+ {
+ name: "empty path",
+ failure: false,
+ container: &pipeline.Container{
+ ID: "content-container",
+ Image: "alpine:latest",
+ },
+ path: "",
+ },
+ }
+
+ // run tests
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ reader, size, err := _engine.PollFileContent(context.Background(), test.container, test.path)
+
+ if test.failure {
+ if err == nil {
+ t.Errorf("PollFileContent should have returned err")
+ }
+
+ return // continue to next test
+ }
+
+ if err != nil {
+ t.Errorf("PollFileContent returned err: %v", err)
+ }
+
+ if test.wantBytes != nil {
+ got, err := io.ReadAll(reader)
+ if err != nil {
+ t.Errorf("failed to read content: %v", err)
+ }
+
+ if string(got) != string(test.wantBytes) {
+ t.Errorf("PollFileContent is %s, want %s", string(got), string(test.wantBytes))
+ }
+
+ if size != int64(len(test.wantBytes)) {
+ t.Errorf("PollFileContent size is %d, want %d", size, int64(len(test.wantBytes)))
+ }
+ }
+ })
+ }
+}
diff --git a/runtime/engine.go b/runtime/engine.go
index e842a9c0..31322c71 100644
--- a/runtime/engine.go
+++ b/runtime/engine.go
@@ -94,4 +94,12 @@ type Engine interface {
// RemoveVolume defines a function that
// deletes the pipeline volume.
RemoveVolume(context.Context, *pipeline.Build) error
+ // Artifact Interface functions
+
+ // PollFileNames defines a function that
+ // captures the artifacts from the pipeline container.
+ PollFileNames(ctx context.Context, ctn *pipeline.Container, _step *pipeline.Container) ([]string, error)
+ // PollFileContent defines a function that
+ // captures the content and size of a file from the pipeline container.
+ PollFileContent(ctx context.Context, ctn *pipeline.Container, path string) (io.Reader, int64, error)
}
diff --git a/runtime/kubernetes/container.go b/runtime/kubernetes/container.go
index 5d795ac0..80544316 100644
--- a/runtime/kubernetes/container.go
+++ b/runtime/kubernetes/container.go
@@ -77,6 +77,22 @@ func (c *client) PollOutputsContainer(_ context.Context, ctn *pipeline.Container
return nil, nil
}
+// PollFileNames grabs test results and attachments from provided path within a container.
+// This is a no-op for kubernetes. Pod environments cannot be dynamic.
+func (c *client) PollFileNames(_ context.Context, ctn *pipeline.Container, _ *pipeline.Container) ([]string, error) {
+ c.Logger.Tracef("no-op: gathering test results and attachments from container %s", ctn.ID)
+
+ return nil, nil
+}
+
+// PollFileContent captures the content and size of a file from the pipeline container.
+// This is a no-op for kubernetes. Pod environments cannot be dynamic.
+func (c *client) PollFileContent(_ context.Context, ctn *pipeline.Container, _ string) (io.Reader, int64, error) {
+ c.Logger.Tracef("no-op: gathering test results and attachments from container %s", ctn.ID)
+
+ return nil, 0, nil
+}
+
// RunContainer creates and starts the pipeline container.
func (c *client) RunContainer(ctx context.Context, ctn *pipeline.Container, _ *pipeline.Build) error {
c.Logger.Tracef("running container %s", ctn.ID)