diff --git a/.gitignore b/.gitignore index 1b32e9a6..5db6e007 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ secrets.env # Dotenv environment file .env +.env_* .env.test # Files to be excluded. diff --git a/.golangci.yml b/.golangci.yml index d6271714..94e5a737 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -125,7 +125,8 @@ linters: gosec: excludes: - G101 # hardcoded credentials - - G117 # exported field of "Secret" style + - G117 + - G705 # exported field of "Secret" style # https://github.com/client9/misspell misspell: locale: US @@ -158,6 +159,7 @@ linters: allow-first-in-block: true allow-whole-block: false branch-max-lines: 2 + exclusions: generated: lax presets: diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index ed973fc3..f4f991b7 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -27,7 +27,10 @@ import ( // //nolint:gocyclo,funlen // ignore cyclomatic complexity and function length func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error { - var err error + var ( + err error + _executor executor.Engine + ) // setup the version v := version.New() @@ -144,8 +147,6 @@ func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error // prepare pipeline by hydrating container ID values based on build information p.Prepare(item.Build.GetRepo().GetOrg(), item.Build.GetRepo().GetName(), item.Build.GetNumber(), false) - logrus.Debugf("setting up exec client with scm token %s with expiration %d", p.Token, p.TokenExp) - // setup exec client with scm token and build token execBuildClient, err = setupExecClient(w.Config.Server, bt.GetToken(), p.Token, p.TokenExp, item.Build) if err != nil { @@ -238,11 +239,13 @@ func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error // setup the executor // // https://pkg.go.dev/github.com/go-vela/worker/executor#New - _executor, err := executor.New(&executor.Setup{ + setup := &executor.Setup{ Logger: logger, Mock: w.Config.Mock, Driver: w.Config.Executor.Driver, MaxLogSize: w.Config.Executor.MaxLogSize, + FileSizeLimit: w.Config.Executor.FileSizeLimit, + BuildFileSizeLimit: w.Config.Executor.BuildFileSizeLimit, LogStreamingTimeout: w.Config.Executor.LogStreamingTimeout, EnforceTrustedRepos: w.Config.Executor.EnforceTrustedRepos, PrivilegedImages: w.Config.Runtime.PrivilegedImages, @@ -253,8 +256,13 @@ func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error Pipeline: p.Sanitize(w.Config.Runtime.Driver), Version: v.Semantic(), OutputCtn: &execOutputCtn, - }) + } + _executor, err = executor.New(setup) + if err != nil { + logger.Errorf("unable to setup executor: %v", err) + return err + } // add the executor to the worker w.Executors[index] = _executor diff --git a/cmd/vela-worker/flags.go b/cmd/vela-worker/flags.go index 23fde260..cfa460dc 100644 --- a/cmd/vela-worker/flags.go +++ b/cmd/vela-worker/flags.go @@ -61,6 +61,18 @@ func flags() []cli.Flag { Sources: cli.EnvVars("WORKER_BUILD_TIMEOUT", "VELA_BUILD_TIMEOUT", "BUILD_TIMEOUT"), Value: 30 * time.Minute, }, + &cli.IntFlag{ + Name: "storage.file-size-limit", + Usage: "maximum file size (in MB) for a single file upload. 0 means no limit.", + Sources: cli.EnvVars("WORKER_STORAGE_FILE_SIZE_LIMIT", "VELA_STORAGE_FILE_SIZE_LIMIT", "STORAGE_FILE_SIZE_LIMIT"), + Value: 100, + }, + &cli.IntFlag{ + Name: "storage.build-file-size-limit", + Usage: "maximum total size (in MB) for all file uploads in a single build. 0 means no limit.", + Sources: cli.EnvVars("WORKER_STORAGE_BUILD_FILE_SIZE_LIMIT", "VELA_STORAGE_BUILD_FILE_SIZE_LIMIT", "STORAGE_BUILD_FILE_SIZE_LIMIT"), + Value: 500, + }, // Logger Flags diff --git a/cmd/vela-worker/run.go b/cmd/vela-worker/run.go index 6d1e419d..3d3c2e4e 100644 --- a/cmd/vela-worker/run.go +++ b/cmd/vela-worker/run.go @@ -105,6 +105,8 @@ func run(ctx context.Context, c *cli.Command) error { Executor: &executor.Setup{ Driver: c.String("executor.driver"), MaxLogSize: c.Uint("executor.max_log_size"), + FileSizeLimit: c.Int("storage.file-size-limit"), + BuildFileSizeLimit: c.Int("storage.build-file-size-limit"), LogStreamingTimeout: c.Duration("executor.log_streaming_timeout"), EnforceTrustedRepos: c.Bool("executor.enforce-trusted-repos"), OutputCtn: outputsCtn, diff --git a/docker-compose.yml b/docker-compose.yml index 0c081b6c..5b464577 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,6 +83,17 @@ services: VELA_DISABLE_WEBHOOK_VALIDATION: 'true' VELA_ENABLE_SECURE_COOKIE: 'false' VELA_REPO_ALLOWLIST: '*' + VELA_SCHEDULE_ALLOWLIST: '*' + VELA_OTEL_TRACING_ENABLE: true + VELA_OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 + VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 100 + VELA_STORAGE_ENABLE: true + VELA_STORAGE_DRIVER: minio + VELA_STORAGE_ADDRESS: 'http://minio:9001' # Address of the MinIO server + VELA_STORAGE_ACCESS_KEY: minioadmin + VELA_STORAGE_SECRET_KEY: minioadmin + VELA_STORAGE_USE_SSL: 'false' + VELA_STORAGE_BUCKET: vela env_file: - .env restart: always @@ -161,5 +172,42 @@ services: cap_add: - IPC_LOCK + # The `minio` compose service hosts the MinIO server instance. + # + # This component is used for storing build artifacts. + # + # https://min.io/ + minio: + container_name: minio + image: minio/minio + restart: always + ports: + - '9001:9001' + - '9002:9002' + networks: + - vela + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin + command: minio server --address ":9001" --console-address ":9002" /data + + # The `minio-setup` compose service is used for setting up the MinIO server instance. + # + # This component is used for creating the bucket and setting permissions. + minio-setup: + image: minio/mc + container_name: minio-setup + depends_on: + - minio + networks: + - vela + entrypoint: > + /bin/sh -c ' + until mc alias set local http://minio:9001 minioadmin minioadmin; do sleep 1; done; + mc mb --ignore-existing local/vela; + mc anonymous set none local/vela; + ' + restart: "no" + networks: vela: \ No newline at end of file diff --git a/executor/linux/build.go b/executor/linux/build.go index d6f8324c..f697fc77 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -556,6 +556,13 @@ func (c *client) ExecBuild(ctx context.Context) error { if c.err != nil { return fmt.Errorf("unable to execute step: %w", c.err) } + + if len(_step.Artifacts.Paths) != 0 { + err := c.outputs.pollFiles(ctx, c.OutputCtn, _step, c.build) + if err != nil { + c.Logger.Errorf("unable to poll files for artifacts: %v", err) + } + } } // create an error group with the context for each stage diff --git a/executor/linux/build_test.go b/executor/linux/build_test.go index 7d7a8456..35f088de 100644 --- a/executor/linux/build_test.go +++ b/executor/linux/build_test.go @@ -190,7 +190,6 @@ func TestLinux_CreateBuild(t *testing.T) { WithBuild(test.build), WithPipeline(_pipeline), WithRuntime(_runtime), - WithVelaClient(_client), ) if err != nil { diff --git a/executor/linux/driver.go b/executor/linux/driver.go index 5ef1dd59..ae4821ea 100644 --- a/executor/linux/driver.go +++ b/executor/linux/driver.go @@ -8,3 +8,8 @@ import "github.com/go-vela/server/constants" func (c *client) Driver() string { return constants.DriverLinux } + +// StorageDriver outputs the configured storage driver. +func (c *client) StorageDriver() string { + return constants.DriverMinio +} diff --git a/executor/linux/linux.go b/executor/linux/linux.go index 83f54dcf..5c85ad3c 100644 --- a/executor/linux/linux.go +++ b/executor/linux/linux.go @@ -29,6 +29,7 @@ type ( Hostname string Version string OutputCtn *pipeline.Container + Uploaded int64 // clients for build actions secret *secretSvc @@ -37,6 +38,8 @@ type ( // private fields init *pipeline.Container maxLogSize uint + fileSizeLimit int64 + buildFileSizeLimit int64 logStreamingTimeout time.Duration privilegedImages []string enforceTrustedRepos bool @@ -74,6 +77,8 @@ func Equal(a, b *client) bool { a.Version == b.Version && reflect.DeepEqual(a.init, b.init) && a.maxLogSize == b.maxLogSize && + a.fileSizeLimit == b.fileSizeLimit && + a.buildFileSizeLimit == b.buildFileSizeLimit && reflect.DeepEqual(a.privilegedImages, b.privilegedImages) && a.enforceTrustedRepos == b.enforceTrustedRepos && reflect.DeepEqual(a.build, b.build) && diff --git a/executor/linux/opts.go b/executor/linux/opts.go index 5c394bae..469870d5 100644 --- a/executor/linux/opts.go +++ b/executor/linux/opts.go @@ -47,6 +47,30 @@ func WithMaxLogSize(size uint) Opt { } } +// WithFileSizeLimit sets the maximum file size (in MB) for a single file upload in the executor client for Linux. +func WithFileSizeLimit(limit int) Opt { + return func(c *client) error { + c.Logger.Trace("configuring file size limit in linux executor client") + + // set the file size limit in the client + c.fileSizeLimit = int64(limit) * 1024 * 1024 + + return nil + } +} + +// WithBuildFileSizeLimit sets the maximum total size (in MB) for all file uploads in a single build in the executor client for Linux. +func WithBuildFileSizeLimit(limit int) Opt { + return func(c *client) error { + c.Logger.Trace("configuring build file size limit in linux executor client") + + // set the build file size limit in the client + c.buildFileSizeLimit = int64(limit) * 1024 * 1024 + + return nil + } +} + // WithLogStreamingTimeout sets the log streaming timeout in the executor client for Linux. func WithLogStreamingTimeout(timeout time.Duration) Opt { return func(c *client) error { diff --git a/executor/linux/opts_test.go b/executor/linux/opts_test.go index 36196330..2f3f1a1a 100644 --- a/executor/linux/opts_test.go +++ b/executor/linux/opts_test.go @@ -70,6 +70,53 @@ func TestLinux_Opt_WithBuild(t *testing.T) { } func TestLinux_Opt_WithMaxLogSize(t *testing.T) { + // setup tests + tests := []struct { + name string + failure bool + fileSizeLimit int + buildFileSizeLimit int + }{ + { + name: "defined", + failure: false, + fileSizeLimit: 200, + buildFileSizeLimit: 1000, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _engine, err := New( + WithFileSizeLimit(test.fileSizeLimit), + WithBuildFileSizeLimit(test.buildFileSizeLimit), + ) + + if test.failure { + if err == nil { + t.Errorf("WithFileSizeLimit should have returned err") + } + + return // continue to next test + } + + if err != nil { + t.Errorf("WithFileSizeLimit returned err: %v", err) + } + + if !reflect.DeepEqual(_engine.fileSizeLimit, int64(test.fileSizeLimit*1024*1024)) { + t.Errorf("WithFileSizeLimit is %v, want %v", _engine.fileSizeLimit, int64(test.fileSizeLimit*1024*1024)) + } + + if !reflect.DeepEqual(_engine.buildFileSizeLimit, int64(test.buildFileSizeLimit*1024*1024)) { + t.Errorf("WithBuildFileSizeLimit is %v, want %v", _engine.buildFileSizeLimit, int64(test.buildFileSizeLimit*1024*1024)) + } + }) + } +} + +func TestLinux_Opt_WithFileSizeLimit(t *testing.T) { // setup tests tests := []struct { name string diff --git a/executor/linux/outputs.go b/executor/linux/outputs.go index b622a859..cf34f632 100644 --- a/executor/linux/outputs.go +++ b/executor/linux/outputs.go @@ -8,11 +8,15 @@ import ( "encoding/base64" "fmt" "maps" + "path/filepath" + "strconv" envparse "github.com/hashicorp/go-envparse" "github.com/sirupsen/logrus" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/storage" ) // outputSvc handles communication with the outputs container during the build. @@ -204,3 +208,97 @@ func (o *outputSvc) poll(ctx context.Context, ctn *pipeline.Container) (map[stri return outputMap, maskMap, nil } + +// pollFiles polls the output for files from the sidecar container. +func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, _step *pipeline.Container, b *api.Build) error { + // exit if outputs container has not been configured + if len(ctn.Image) == 0 { + return fmt.Errorf("no outputs container configured") + } + + // update engine logger with outputs metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus#Entry.WithField + logger := o.client.Logger.WithField("artifact-outputs", ctn.Name) + + creds, res, err := o.client.Vela.Build.GetSTSCreds(ctx, b.GetRepo().GetOrg(), b.GetRepo().GetName(), + b.GetNumber()) + if err != nil { + return fmt.Errorf("unable to get sts storage creds %w with response code %d", err, res.StatusCode) + } + + stsStorageClient, err := storage.New(&storage.Setup{ + Enable: creds.Enable, + Driver: creds.Driver, + Endpoint: creds.Endpoint, + AccessKey: creds.AccessKey, + SecretKey: creds.SecretKey, + Bucket: creds.Bucket, + Secure: creds.Secure, + Token: creds.SessionToken, + }) + if err != nil { + return fmt.Errorf("unable to create sts storage client %w with response code %d", err, res.StatusCode) + } + + if stsStorageClient == nil { + return fmt.Errorf("sts storage client is nil") + } + + // grab file paths from the container + filesPath, err := o.client.Runtime.PollFileNames(ctx, ctn, _step) + if err != nil { + return fmt.Errorf("unable to poll file names: %w", err) + } + + if len(filesPath) == 0 { + return fmt.Errorf("no files found for file list: %v", _step.Artifacts.Paths) + } + + // process each file found + for _, filePath := range filesPath { + fileName := filepath.Base(filePath) + logger.Debugf("processing file: %s (path: %s)", fileName, filePath) + + // get file content from container + reader, size, err := o.client.Runtime.PollFileContent(ctx, ctn, filePath) + if err != nil { + return fmt.Errorf("unable to poll file content for %s: %w", filePath, err) + } + + // TODO: surface this skip to the user + if o.client.fileSizeLimit > 0 && size > o.client.fileSizeLimit { + logger.Infof("skipping file %s due to file size limit", filePath) + continue + } + + if o.client.buildFileSizeLimit > 0 && size+o.client.Uploaded > o.client.buildFileSizeLimit { + logger.Infof("skipping file %s due to build file size limit", filePath) + continue + } + + // create storage object path + objectName := fmt.Sprintf("%s/%s/%s/%s", + b.GetRepo().GetOrg(), + b.GetRepo().GetName(), + strconv.FormatInt(b.GetNumber(), 10), + fileName) + + obj := &api.Object{ + Bucket: api.Bucket{ + BucketName: creds.Bucket, + }, + ObjectName: objectName, + FilePath: filePath, + } + // upload file to storage + err = stsStorageClient.UploadObject(ctx, obj, reader, size) + if err != nil { + return fmt.Errorf("unable to upload object %s: %w", fileName, err) + } + + o.client.Uploaded += size + } + + return nil +} diff --git a/executor/linux/stage.go b/executor/linux/stage.go index 41399b37..fcb99236 100644 --- a/executor/linux/stage.go +++ b/executor/linux/stage.go @@ -198,6 +198,13 @@ func (c *client) ExecStage(ctx context.Context, s *pipeline.Stage, m *sync.Map) if _step.ExitCode != 0 && !_step.Ruleset.Continue { stageStatus = constants.StatusFailure } + + if len(_step.Artifacts.Paths) != 0 { + err := c.outputs.pollFiles(ctx, c.OutputCtn, _step, c.build) + if err != nil { + c.Logger.Errorf("unable to poll files for artifacts: %v", err) + } + } } return nil diff --git a/executor/local/service.go b/executor/local/service.go index 34186a03..aded9ace 100644 --- a/executor/local/service.go +++ b/executor/local/service.go @@ -119,8 +119,6 @@ func (c *client) StreamService(ctx context.Context, ctn *pipeline.Container) err // scan entire container output for scanner.Scan() { // ensure we output to stdout - // - //nolint:gosec // false positive fmt.Fprintln(c.stdout, _pattern, scanner.Text()) } diff --git a/executor/local/step.go b/executor/local/step.go index dc70b054..ac12a79d 100644 --- a/executor/local/step.go +++ b/executor/local/step.go @@ -195,8 +195,6 @@ func (c *client) StreamStep(ctx context.Context, ctn *pipeline.Container) error // scan entire container output for scanner.Scan() { // ensure we output to stdout - // - //nolint:gosec // false positive fmt.Fprintln(c.stdout, _pattern, scanner.Text()) } diff --git a/executor/setup.go b/executor/setup.go index bd2ada04..a776e4ea 100644 --- a/executor/setup.go +++ b/executor/setup.go @@ -34,6 +34,10 @@ type Setup struct { Driver string // specifies the maximum log size MaxLogSize uint + // specifies file size limit for artifacts + FileSizeLimit int + // specifies build file size limit for artifacts + BuildFileSizeLimit int // specifies how long to wait after the build finishes // for log streaming to complete LogStreamingTimeout time.Duration @@ -75,12 +79,12 @@ func (s *Setup) Darwin() (Engine, error) { func (s *Setup) Linux() (Engine, error) { logrus.Trace("creating linux executor client from setup") - // create new Linux executor engine - // - // https://pkg.go.dev/github.com/go-vela/worker/executor/linux#New - return linux.New( + // create options for Linux executor + opts := []linux.Opt{ linux.WithBuild(s.Build), linux.WithMaxLogSize(s.MaxLogSize), + linux.WithFileSizeLimit(s.FileSizeLimit), + linux.WithBuildFileSizeLimit(s.BuildFileSizeLimit), linux.WithLogStreamingTimeout(s.LogStreamingTimeout), linux.WithPrivilegedImages(s.PrivilegedImages), linux.WithEnforceTrustedRepos(s.EnforceTrustedRepos), @@ -91,7 +95,11 @@ func (s *Setup) Linux() (Engine, error) { linux.WithVersion(s.Version), linux.WithLogger(s.Logger), linux.WithOutputCtn(s.OutputCtn), - ) + } + // create new Linux executor engine + // + // https://pkg.go.dev/github.com/go-vela/worker/executor/linux#New + return linux.New(opts...) } // Local creates and returns a Vela engine capable of @@ -99,10 +107,7 @@ func (s *Setup) Linux() (Engine, error) { func (s *Setup) Local() (Engine, error) { logrus.Trace("creating local executor client from setup") - // create new Local executor engine - // - // https://pkg.go.dev/github.com/go-vela/worker/executor/local#New - return local.New( + opts := []local.Opt{ local.WithBuild(s.Build), local.WithHostname(s.Hostname), local.WithPipeline(s.Pipeline), @@ -111,7 +116,12 @@ func (s *Setup) Local() (Engine, error) { local.WithVersion(s.Version), local.WithMockStdout(s.Mock), local.WithOutputCtn(s.OutputCtn), - ) + } + + // create new Local executor engine + // + // https://pkg.go.dev/github.com/go-vela/worker/executor/local#New + return local.New(opts...) } // Windows creates and returns a Vela engine capable of diff --git a/executor/setup_test.go b/executor/setup_test.go index e89700a7..c48e7f26 100644 --- a/executor/setup_test.go +++ b/executor/setup_test.go @@ -72,6 +72,8 @@ func TestExecutor_Setup_Linux(t *testing.T) { want, err := linux.New( linux.WithBuild(_build), linux.WithMaxLogSize(2097152), + linux.WithFileSizeLimit(10), + linux.WithBuildFileSizeLimit(100), linux.WithLogStreamingTimeout(1*time.Second), linux.WithHostname("localhost"), linux.WithPipeline(_pipeline), @@ -84,14 +86,16 @@ func TestExecutor_Setup_Linux(t *testing.T) { } _setup := &Setup{ - Build: _build, - Client: _client, - Driver: constants.DriverLinux, - MaxLogSize: 2097152, - Hostname: "localhost", - Pipeline: _pipeline, - Runtime: _runtime, - Version: "v1.0.0", + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + MaxLogSize: 2097152, + FileSizeLimit: 10, + BuildFileSizeLimit: 100, + Hostname: "localhost", + Pipeline: _pipeline, + Runtime: _runtime, + Version: "v1.0.0", } // run test diff --git a/mock/docker/container.go b/mock/docker/container.go index 419ab638..2f2ac60a 100644 --- a/mock/docker/container.go +++ b/mock/docker/container.go @@ -4,12 +4,14 @@ package docker import ( "archive/tar" + "bufio" "bytes" "context" "encoding/json" "errors" "fmt" "io" + "net" "strings" "time" @@ -24,6 +26,19 @@ import ( v1 "github.com/opencontainers/image-spec/specs-go/v1" ) +// mockConn is a mock implementation of net.Conn for testing purposes. +// It provides no-op implementations of all net.Conn methods. +type mockConn struct{} + +func (m *mockConn) Read(b []byte) (n int, err error) { return 0, io.EOF } +func (m *mockConn) Write(b []byte) (n int, err error) { return len(b), nil } +func (m *mockConn) Close() error { return nil } +func (m *mockConn) LocalAddr() net.Addr { return nil } +func (m *mockConn) RemoteAddr() net.Addr { return nil } +func (m *mockConn) SetDeadline(t time.Time) error { return nil } +func (m *mockConn) SetReadDeadline(t time.Time) error { return nil } +func (m *mockConn) SetWriteDeadline(t time.Time) error { return nil } + // ContainerService implements all the container // related functions for the Docker mock. type ContainerService struct{} @@ -106,8 +121,51 @@ func (c *ContainerService) ContainerDiff(_ context.Context, _ string) ([]contain // running inside a Docker container. // // https://pkg.go.dev/github.com/docker/docker/client#Client.ContainerExecAttach -func (c *ContainerService) ContainerExecAttach(_ context.Context, _ string, _ container.ExecAttachOptions) (types.HijackedResponse, error) { - return types.HijackedResponse{}, nil +func (c *ContainerService) ContainerExecAttach(_ context.Context, execID string, _ container.ExecAttachOptions) (types.HijackedResponse, error) { + // create a buffer to hold mock output + var buf bytes.Buffer + + // write mock stdout using stdcopy format + stdoutWriter := stdcopy.NewStdWriter(&buf, stdcopy.Stdout) + stderrWriter := stdcopy.NewStdWriter(&buf, stdcopy.Stderr) + + // check for specific test scenarios based on execID + if strings.Contains(execID, "error") { + // write to stderr to simulate an error + _, _ = stderrWriter.Write([]byte("mock exec error")) + } else if strings.Contains(execID, "multiline") { + // write multiple lines for testing + _, _ = stdoutWriter.Write([]byte("line1\nline2\nline3")) + } else if strings.Contains(execID, "artifacts-find") { + // simulate find command for artifacts - return file paths + _, _ = stdoutWriter.Write([]byte("/vela/artifacts/test_results/alpha.txt\n/vela/artifacts/test_results/beta.txt\n/vela/artifacts/build_results/alpha.txt\n/vela/artifacts/build_results/beta.txt")) + } else if strings.Contains(execID, "test-results-xml") { + // simulate find command for test results XML files + _, _ = stdoutWriter.Write([]byte("/vela/workspace/test-results/junit.xml\n/vela/workspace/test-results/report.xml")) + } else if strings.Contains(execID, "cypress-screenshots") { + // simulate find command for cypress screenshots + _, _ = stdoutWriter.Write([]byte("/vela/workspace/cypress/screenshots/test1/screenshot1.png\n/vela/workspace/cypress/screenshots/test1/screenshot2.png\n/vela/workspace/cypress/screenshots/test2/error.png")) + } else if strings.Contains(execID, "cypress-videos") { + // simulate find command for cypress videos + _, _ = stdoutWriter.Write([]byte("/vela/workspace/cypress/videos/test1.mp4\n/vela/workspace/cypress/videos/test2.mp4")) + } else if strings.Contains(execID, "cypress-all") { + // simulate find command for all cypress artifacts (screenshots + videos) + _, _ = stdoutWriter.Write([]byte("/vela/workspace/cypress/screenshots/test1/screenshot1.png\n/vela/workspace/cypress/screenshots/test2/error.png\n/vela/workspace/cypress/videos/test1.mp4\n/vela/workspace/cypress/videos/test2.mp4")) + } else if strings.Contains(execID, "not-found") { + // simulate path not found + _, _ = stderrWriter.Write([]byte("find: '/not-found': No such file or directory")) + } else { + // default mock output + _, _ = stdoutWriter.Write([]byte("mock exec output")) + } + + // create a HijackedResponse with the mock data + response := types.HijackedResponse{ + Reader: bufio.NewReader(&buf), + Conn: &mockConn{}, // Use mock connection to avoid nil pointer dereference + } + + return response, nil } // ContainerExecCreate is a helper function to simulate @@ -115,8 +173,52 @@ func (c *ContainerService) ContainerExecAttach(_ context.Context, _ string, _ co // Docker container. // // https://pkg.go.dev/github.com/docker/docker/client#Client.ContainerExecCreate -func (c *ContainerService) ContainerExecCreate(_ context.Context, _ string, _ container.ExecOptions) (container.ExecCreateResponse, error) { - return container.ExecCreateResponse{}, nil +func (c *ContainerService) ContainerExecCreate(_ context.Context, ctn string, config container.ExecOptions) (container.ExecCreateResponse, error) { + // verify a container was provided + if len(ctn) == 0 { + return container.ExecCreateResponse{}, errors.New("no container provided") + } + + // check if the container is not found + if strings.Contains(ctn, "notfound") || strings.Contains(ctn, "not-found") { + return container.ExecCreateResponse{}, errdefs.NotFound(fmt.Errorf("Error: No such container: %s", ctn)) + } + + // create exec ID based on command for testing scenarios + execID := stringid.GenerateRandomID() + + // check command for specific test scenarios + if len(config.Cmd) > 0 { + cmdStr := strings.Join(config.Cmd, " ") + if strings.Contains(cmdStr, "error") { + execID = "exec-error-" + execID + } else if strings.Contains(cmdStr, "multiline") { + execID = "exec-multiline-" + execID + } else if strings.Contains(cmdStr, "find") { + // For artifact file search commands + if strings.Contains(cmdStr, "/not-found") { + execID = "exec-not-found-" + execID + } else if strings.Contains(cmdStr, "artifacts") { + execID = "exec-artifacts-find-" + execID + } else if strings.Contains(cmdStr, "test-results") && strings.Contains(cmdStr, ".xml") { + execID = "exec-test-results-xml-" + execID + } else if strings.Contains(cmdStr, "cypress/screenshots") && strings.Contains(cmdStr, ".png") { + execID = "exec-cypress-screenshots-" + execID + } else if strings.Contains(cmdStr, "cypress/videos") && strings.Contains(cmdStr, ".mp4") { + execID = "exec-cypress-videos-" + execID + } else if strings.Contains(cmdStr, "cypress") { + // Generic cypress pattern for combined searches + execID = "exec-cypress-all-" + execID + } + } + } + + // create response object to return + response := container.ExecCreateResponse{ + ID: execID, + } + + return response, nil } // ContainerExecInspect is a helper function to simulate @@ -523,7 +625,19 @@ func (c *ContainerService) CopyFromContainer(_ context.Context, _ string, path s tw := tar.NewWriter(&buf) - content := []byte("key=value") + // Determine content based on path for test scenarios + var content []byte + if strings.Contains(path, "artifacts") && strings.Contains(path, "alpha.txt") { + content = []byte("results") + } else if strings.Contains(path, "test-results") && strings.Contains(path, ".xml") { + content = []byte("") + } else if strings.Contains(path, "cypress/screenshots") && strings.Contains(path, ".png") { + content = []byte("PNG_BINARY_DATA") + } else if strings.Contains(path, "cypress/videos") && strings.Contains(path, ".mp4") { + content = []byte("MP4_BINARY_DATA") + } else { + content = []byte("key=value") + } hdr := &tar.Header{ Name: path, diff --git a/runtime/docker/artifact.go b/runtime/docker/artifact.go new file mode 100644 index 00000000..cccbead4 --- /dev/null +++ b/runtime/docker/artifact.go @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "fmt" + "io" + "path/filepath" + "strings" + + "github.com/containerd/errdefs" + dockerContainerTypes "github.com/docker/docker/api/types/container" + "github.com/docker/docker/pkg/stdcopy" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +// execContainerLines runs `sh -c cmd` in the named container and +// returns its stdout split by newline (error if anything on stderr). +func (c *client) execContainerLines(ctx context.Context, containerID, cmd string) ([]string, error) { + execConfig := dockerContainerTypes.ExecOptions{ + Tty: true, + Cmd: []string{"sh", "-c", cmd}, + AttachStdout: true, + AttachStderr: true, + } + + resp, err := c.Docker.ContainerExecCreate(ctx, containerID, execConfig) + if err != nil { + return nil, fmt.Errorf("create exec: %w", err) + } + + attach, err := c.Docker.ContainerExecAttach(ctx, resp.ID, dockerContainerTypes.ExecAttachOptions{}) + if err != nil { + return nil, fmt.Errorf("attach exec: %w", err) + } + + defer attach.Close() + + var outBuf, errBuf bytes.Buffer + if _, err := stdcopy.StdCopy(&outBuf, &errBuf, attach.Reader); err != nil { + return nil, fmt.Errorf("copy exec output: %w", err) + } + + if errBuf.Len() > 0 { + return nil, fmt.Errorf("exec error: %s", errBuf.String()) + } + + lines := strings.Split(strings.TrimSpace(outBuf.String()), "\n") + + return lines, nil +} + +// PollFileNames searches for files matching the provided patterns within a container. +func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, _step *pipeline.Container) ([]string, error) { + c.Logger.Tracef("gathering files from container %s", ctn.ID) + + if ctn.Image == "" { + return nil, nil + } + + var results []string + + paths := _step.Artifacts.Paths + + for _, pattern := range paths { + // use find command to locate files matching the pattern + cmd := fmt.Sprintf("find %s -type f -path '*%s' -print", _step.Environment["VELA_WORKSPACE"], pattern) + c.Logger.Debugf("searching for files with pattern: %s", pattern) + + lines, err := c.execContainerLines(ctx, ctn.ID, cmd) + if err != nil { + return nil, fmt.Errorf("failed to search for pattern %q: %w", pattern, err) + } + + c.Logger.Tracef("found %d candidates for pattern %s", len(lines), pattern) + + // process each found file + for _, line := range lines { + filePath := filepath.Clean(strings.TrimSpace(line)) + if filePath == "" { + continue + } + + c.Logger.Debugf("accepted file: %s", filePath) + results = append(results, filePath) + } + } + + if len(results) == 0 { + return results, fmt.Errorf("no matching files found for patterns: %v", paths) + } + + return results, nil +} + +// PollFileContent retrieves the content and size of a file inside a container. +func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, path string) (io.Reader, int64, error) { + c.Logger.Tracef("gathering test results and attachments from container %s", ctn.ID) + + if len(ctn.Image) == 0 || len(path) == 0 { + return nil, 0, nil + } + + // copy file from outputs container + reader, _, err := c.Docker.CopyFromContainer(ctx, ctn.ID, path) + if err != nil { + c.Logger.Debugf("PollFileContent CopyFromContainer failed for %q: %v", path, err) + // early non-error exit if not found + if errdefs.IsNotFound(err) { + return nil, 0, nil + } + + return nil, 0, err + } + + defer reader.Close() + + // docker returns a tar archive for the path + tr := tar.NewReader(reader) + + header, err := tr.Next() + if err != nil { + // if the tar has no entries or is finished unexpectedly + if errors.Is(err, io.EOF) { + c.Logger.Debugf("PollFileContent: no tar entries for %q", path) + + return nil, 0, nil + } + + c.Logger.Debugf("PollFileContent tr.Next failed for %q: %v", path, err) + + return nil, 0, err + } + + // Ensure the tar entry is a regular file (not dir, symlink, etc.) + if header.Typeflag != tar.TypeReg { + c.Logger.Debugf("PollFileContent unexpected tar entry type %v for %q", header.Typeflag, path) + + return nil, 0, fmt.Errorf("unexpected tar entry type %v for %q", header.Typeflag, path) + } + + // Read file contents. Use io.ReadAll to avoid dealing with CopyN EOF nuances. + fileBytes, err := io.ReadAll(tr) + if err != nil { + c.Logger.Debugf("PollFileContent ReadAll failed for %q: %v", path, err) + + return nil, 0, err + } + + if len(fileBytes) == 0 { + c.Logger.Errorf("PollFileContent returned no data for path: %s", path) + + return nil, 0, fmt.Errorf("no data returned from container for %q", path) + } + + // Return a reader and length (use int64 for size) + return bytes.NewReader(fileBytes), int64(len(fileBytes)), nil +} diff --git a/runtime/docker/artifact_test.go b/runtime/docker/artifact_test.go new file mode 100644 index 00000000..48887f6a --- /dev/null +++ b/runtime/docker/artifact_test.go @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "context" + "io" + "testing" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestDocker_PollFileNames(t *testing.T) { + // setup Docker + _engine, err := NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + name string + failure bool + container *pipeline.Container + step *pipeline.Container + wantFiles int + }{ + { + name: "artifacts directory search", + failure: false, + container: &pipeline.Container{ + ID: "artifacts-container", + Image: "alpine:latest", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{"artifacts/*/*.txt"}, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/vela/workspace", + }, + }, + wantFiles: 4, // alpha.txt and beta.txt in test_results and build_results + }, + { + name: "test-results XML files", + failure: false, + container: &pipeline.Container{ + ID: "test-results-container", + Image: "alpine:latest", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{"test-results/*.xml"}, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/vela/workspace", + }, + }, + wantFiles: 2, // junit.xml and report.xml + }, + { + name: "cypress screenshots PNG files", + failure: false, + container: &pipeline.Container{ + ID: "cypress-screenshots-container", + Image: "cypress/browsers:latest", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{"cypress/screenshots/**/*.png"}, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/vela/workspace", + }, + }, + wantFiles: 3, // screenshot1.png, screenshot2.png, error.png + }, + { + name: "cypress videos MP4 files", + failure: false, + container: &pipeline.Container{ + ID: "cypress-videos-container", + Image: "cypress/browsers:latest", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{"cypress/videos/**/*.mp4"}, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/vela/workspace", + }, + }, + wantFiles: 2, // test1.mp4, test2.mp4 + }, + { + name: "multiple cypress patterns", + failure: false, + container: &pipeline.Container{ + ID: "cypress-all-container", + Image: "cypress/browsers:latest", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{ + "cypress/screenshots/**/*.png", + "cypress/videos/**/*.mp4", + }, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/vela/workspace", + }, + }, + wantFiles: 5, // 3 PNG screenshots + 2 MP4 videos + }, + { + name: "combined test-results and cypress artifacts", + failure: false, + container: &pipeline.Container{ + ID: "combined-artifacts-container", + Image: "cypress/browsers:latest", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{ + "test-results/*.xml", + "cypress/screenshots/**/*.png", + "cypress/videos/**/*.mp4", + }, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/vela/workspace", + }, + }, + wantFiles: 7, // 2 XML + 3 PNG + 2 MP4 + }, + { + name: "directory not found", + failure: true, + container: &pipeline.Container{ + ID: "artifacts-container", + Image: "alpine:latest", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{"artifacts/*.txt"}, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/not-found", + }, + }, + }, + { + name: "empty container image", + failure: false, + container: &pipeline.Container{ + ID: "no-image", + Image: "", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{"*.txt"}, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/vela/workspace", + }, + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := _engine.PollFileNames(context.Background(), test.container, test.step) + + if test.failure { + if err == nil { + t.Errorf("PollFileNames should have returned err") + } + + return // continue to next test + } + + if err != nil { + t.Errorf("PollFileNames returned err: %v", err) + } + + if test.wantFiles > 0 && len(got) != test.wantFiles { + t.Errorf("PollFileNames returned %d files, want %d", len(got), test.wantFiles) + } + }) + } +} + +func TestDocker_PollFileContent(t *testing.T) { + // setup Docker + _engine, err := NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + name string + failure bool + container *pipeline.Container + path string + wantBytes []byte + }{ + { + name: "file content from default path", + failure: false, + container: &pipeline.Container{ + ID: "content-container", + Image: "alpine:latest", + }, + path: "/vela/artifacts/test_results/alpha.txt", + wantBytes: []byte("results"), + }, + { + name: "test-results XML file content", + failure: false, + container: &pipeline.Container{ + ID: "content-container", + Image: "alpine:latest", + }, + path: "/vela/workspace/test-results/junit.xml", + wantBytes: []byte(""), + }, + { + name: "cypress screenshot PNG file", + failure: false, + container: &pipeline.Container{ + ID: "content-container", + Image: "cypress/browsers:latest", + }, + path: "/vela/workspace/cypress/screenshots/test1/screenshot1.png", + wantBytes: []byte("PNG_BINARY_DATA"), + }, + { + name: "cypress video MP4 file", + failure: false, + container: &pipeline.Container{ + ID: "content-container", + Image: "cypress/browsers:latest", + }, + path: "/vela/workspace/cypress/videos/test1.mp4", + wantBytes: []byte("MP4_BINARY_DATA"), + }, + { + name: "path not found", + failure: false, + container: &pipeline.Container{ + ID: "content-container", + Image: "alpine:latest", + }, + path: "not-found", + }, + { + name: "empty container image", + failure: false, + container: new(pipeline.Container), + path: "/some/path", + }, + { + name: "empty path", + failure: false, + container: &pipeline.Container{ + ID: "content-container", + Image: "alpine:latest", + }, + path: "", + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + reader, size, err := _engine.PollFileContent(context.Background(), test.container, test.path) + + if test.failure { + if err == nil { + t.Errorf("PollFileContent should have returned err") + } + + return // continue to next test + } + + if err != nil { + t.Errorf("PollFileContent returned err: %v", err) + } + + if test.wantBytes != nil { + got, err := io.ReadAll(reader) + if err != nil { + t.Errorf("failed to read content: %v", err) + } + + if string(got) != string(test.wantBytes) { + t.Errorf("PollFileContent is %s, want %s", string(got), string(test.wantBytes)) + } + + if size != int64(len(test.wantBytes)) { + t.Errorf("PollFileContent size is %d, want %d", size, int64(len(test.wantBytes))) + } + } + }) + } +} diff --git a/runtime/engine.go b/runtime/engine.go index e842a9c0..31322c71 100644 --- a/runtime/engine.go +++ b/runtime/engine.go @@ -94,4 +94,12 @@ type Engine interface { // RemoveVolume defines a function that // deletes the pipeline volume. RemoveVolume(context.Context, *pipeline.Build) error + // Artifact Interface functions + + // PollFileNames defines a function that + // captures the artifacts from the pipeline container. + PollFileNames(ctx context.Context, ctn *pipeline.Container, _step *pipeline.Container) ([]string, error) + // PollFileContent defines a function that + // captures the content and size of a file from the pipeline container. + PollFileContent(ctx context.Context, ctn *pipeline.Container, path string) (io.Reader, int64, error) } diff --git a/runtime/kubernetes/container.go b/runtime/kubernetes/container.go index 5d795ac0..80544316 100644 --- a/runtime/kubernetes/container.go +++ b/runtime/kubernetes/container.go @@ -77,6 +77,22 @@ func (c *client) PollOutputsContainer(_ context.Context, ctn *pipeline.Container return nil, nil } +// PollFileNames grabs test results and attachments from provided path within a container. +// This is a no-op for kubernetes. Pod environments cannot be dynamic. +func (c *client) PollFileNames(_ context.Context, ctn *pipeline.Container, _ *pipeline.Container) ([]string, error) { + c.Logger.Tracef("no-op: gathering test results and attachments from container %s", ctn.ID) + + return nil, nil +} + +// PollFileContent captures the content and size of a file from the pipeline container. +// This is a no-op for kubernetes. Pod environments cannot be dynamic. +func (c *client) PollFileContent(_ context.Context, ctn *pipeline.Container, _ string) (io.Reader, int64, error) { + c.Logger.Tracef("no-op: gathering test results and attachments from container %s", ctn.ID) + + return nil, 0, nil +} + // RunContainer creates and starts the pipeline container. func (c *client) RunContainer(ctx context.Context, ctn *pipeline.Container, _ *pipeline.Build) error { c.Logger.Tracef("running container %s", ctn.ID)