From aad77ed6638dc4f4e1326ac9ebb3807953f1082e Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Mon, 7 Apr 2025 14:19:48 -0500 Subject: [PATCH 01/63] test report execution logic --- cmd/vela-worker/run.go | 2 +- cmd/vela-worker/worker.go | 2 +- executor/executor_test.go | 6 +- executor/linux/build.go | 23 +++- executor/linux/linux.go | 5 +- executor/linux/linux_test.go | 6 +- executor/linux/opts.go | 26 +++++ executor/linux/outputs.go | 69 ++++++++++++ executor/local/local_test.go | 6 +- executor/setup.go | 9 ++ executor/setup_test.go | 14 +++ internal/build/snapshot_test.go | 6 +- internal/build/upload_test.go | 6 +- internal/service/snapshot_test.go | 10 +- internal/service/upload_test.go | 10 +- internal/step/skip_test.go | 26 ++--- internal/step/snapshot_test.go | 20 ++-- runtime/docker/container.go | 2 +- runtime/docker/test_report.go | 170 +++++++++++++++++++++++++++++ runtime/docker/test_report_test.go | 76 +++++++++++++ runtime/engine.go | 8 ++ runtime/kubernetes/container.go | 18 ++- 22 files changed, 465 insertions(+), 55 deletions(-) create mode 100644 runtime/docker/test_report.go create mode 100644 runtime/docker/test_report_test.go diff --git a/cmd/vela-worker/run.go b/cmd/vela-worker/run.go index dbe5e287..6b397b20 100644 --- a/cmd/vela-worker/run.go +++ b/cmd/vela-worker/run.go @@ -95,7 +95,7 @@ func run(c *cli.Context) error { }, // build configuration Build: &Build{ - Limit: c.Int("build.limit"), + Limit: c.Int64("build.limit"), Timeout: c.Duration("build.timeout"), }, // build configuration diff --git a/cmd/vela-worker/worker.go b/cmd/vela-worker/worker.go index 834a88b0..85ac6d5b 100644 --- a/cmd/vela-worker/worker.go +++ b/cmd/vela-worker/worker.go @@ -22,7 +22,7 @@ type ( // Build represents the worker configuration for build information. Build struct { - Limit int + Limit int64 Timeout time.Duration } diff --git a/executor/executor_test.go b/executor/executor_test.go index 8282d75f..48b0c6a5 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -260,7 +260,7 @@ var ( Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -271,9 +271,9 @@ var ( _build = &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), diff --git a/executor/linux/build.go b/executor/linux/build.go index 437b225b..a5f20a89 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -29,6 +29,11 @@ func (c *client) CreateBuild(ctx context.Context) error { // // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Snapshot defer func() { build.Snapshot(c.build, c.Vela, c.err, c.Logger) }() + // Check if storage client is initialized + // and if storage is enable + if c.Storage == nil { + return fmt.Errorf("storage client is not initialized") + } // update the build fields c.build.SetStatus(constants.StatusRunning) @@ -396,7 +401,7 @@ func (c *client) AssembleBuild(ctx context.Context) error { } // create outputs container with a timeout equal to the repo timeout - c.err = c.outputs.create(ctx, c.OutputCtn, (int64(60) * c.build.GetRepo().GetTimeout())) + c.err = c.outputs.create(ctx, c.OutputCtn, int64(60*c.build.GetRepo().GetTimeout())) if c.err != nil { return fmt.Errorf("unable to create outputs container: %w", c.err) } @@ -543,6 +548,22 @@ func (c *client) ExecBuild(ctx context.Context) error { _step.Secrets = append(_step.Secrets, sec) } + // logic for polling files only if the test-report step is present + // iterate through the steps in the build + for _, s := range c.pipeline.Steps { + c.Logger.Infof("polling files for %s step", s.Name) + + if len(s.TestReport.Results) != 0 { + err := c.outputs.pollFiles(ctx, c.OutputCtn, s.TestReport.Results) + c.Logger.Errorf("unable to poll files for results: %v", err) + } + if len(s.TestReport.Attachments) != 0 { + err := c.outputs.pollFiles(ctx, c.OutputCtn, s.TestReport.Attachments) + c.Logger.Errorf("unable to poll files for attachments: %v", err) + } + + } + // perform any substitution on dynamic variables err = _step.Substitute() if err != nil { diff --git a/executor/linux/linux.go b/executor/linux/linux.go index 70502d85..188b08e0 100644 --- a/executor/linux/linux.go +++ b/executor/linux/linux.go @@ -8,13 +8,13 @@ import ( "sync" "time" - "github.com/sirupsen/logrus" - "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime" + "github.com/sirupsen/logrus" ) type ( @@ -29,6 +29,7 @@ type ( Hostname string Version string OutputCtn *pipeline.Container + Storage storage.Storage // clients for build actions secret *secretSvc diff --git a/executor/linux/linux_test.go b/executor/linux/linux_test.go index d6c7d0fd..9a026c87 100644 --- a/executor/linux/linux_test.go +++ b/executor/linux/linux_test.go @@ -168,9 +168,9 @@ func TestLinux_New(t *testing.T) { func testBuild() *api.Build { return &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: testRepo(), - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), @@ -206,7 +206,7 @@ func testRepo() *api.Repo { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), diff --git a/executor/linux/opts.go b/executor/linux/opts.go index 5c394bae..397e106c 100644 --- a/executor/linux/opts.go +++ b/executor/linux/opts.go @@ -4,6 +4,7 @@ package linux import ( "fmt" + "github.com/go-vela/server/storage" "time" "github.com/sirupsen/logrus" @@ -209,3 +210,28 @@ func withStreamRequests(s chan message.StreamRequest) Opt { return nil } } + +// WithStorage sets the storage in the executor client for Linux. +func WithStorage(s *storage.Setup) Opt { + return func(c *client) error { + c.Logger.Trace("configuring storage in linux executor client") + + // check if the storage provided is empty + if &s == nil { + return fmt.Errorf("empty storage setup provided") + } + + // set the storage in the client + var err error + c.Storage, err = storage.New(s) + if err != nil { + return fmt.Errorf("unable to create storage: %v", err) + + } + if c.Storage == nil { + return fmt.Errorf("empty storage setup: %v", err) + } + + return nil + } +} diff --git a/executor/linux/outputs.go b/executor/linux/outputs.go index 79764b93..564b8015 100644 --- a/executor/linux/outputs.go +++ b/executor/linux/outputs.go @@ -7,6 +7,7 @@ import ( "context" "encoding/base64" "fmt" + api "github.com/go-vela/server/api/types" envparse "github.com/hashicorp/go-envparse" "github.com/sirupsen/logrus" @@ -148,3 +149,71 @@ func (o *outputSvc) poll(ctx context.Context, ctn *pipeline.Container) (map[stri return outputMap, maskMap, nil } + +// pollFiles tails the output for sidecar container. +func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, fileList []string) error { + // exit if outputs container has not been configured + if len(ctn.Image) == 0 { + return nil + } + + // update engine logger with outputs metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus#Entry.WithField + logger := o.client.Logger.WithField("test-outputs", ctn.Name) + + logger.Debug("tailing container") + logger.Debugf("fileList: %v", fileList) + // grab outputs + fileNames, err := o.client.Runtime.PollFileNames(ctx, ctn, fileList) + if err != nil { + return fmt.Errorf("unable to poll file names: %v", err) + } + if len(fileNames) != 0 { + for _, fileName := range fileNames { + logger.Infof("fileName: %v", fileName) + reader, size, err := o.client.Runtime.PollFileContent(ctx, ctn, fileName) + if err != nil { + logger.Errorf("unable to poll file content: %v", err) + return err + } + + err = o.client.Storage.UploadObject(ctx, &api.Object{ + ObjectName: fileName, + Bucket: api.Bucket{BucketName: o.client.Storage.GetBucket(ctx)}, + FilePath: fileName, + }, reader, size) + if err != nil { + logger.Errorf("unable to upload object: %v", err) + return err + } + + } + logger.Infof("fileNames: %v", fileNames) + return nil + } + logger.Debug("no files found") + return fmt.Errorf("no files found: %v", err) + + //reader := bytes.NewReader(outputBytes) + // + //outputMap, err := envparse.Parse(reader) + //if err != nil { + // logger.Debugf("unable to parse output map: %v", err) + //} + // + //// grab masked outputs + //maskedBytes, err := o.client.Runtime.PollOutputsContainer(ctx, ctn, "/vela/outputs/masked.env") + //if err != nil { + // return nil, nil, err + //} + // + //reader = bytes.NewReader(maskedBytes) + // + //maskMap, err := envparse.Parse(reader) + //if err != nil { + // logger.Debugf("unable to parse masked output map: %v", err) + //} + + //return outputMap, maskMap, nil +} diff --git a/executor/local/local_test.go b/executor/local/local_test.go index 26c7c9cc..4479920b 100644 --- a/executor/local/local_test.go +++ b/executor/local/local_test.go @@ -164,9 +164,9 @@ func TestLocal_New(t *testing.T) { func testBuild() *api.Build { return &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: testRepo(), - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), @@ -202,7 +202,7 @@ func testRepo() *api.Repo { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), diff --git a/executor/setup.go b/executor/setup.go index bd2ada04..ef1f1280 100644 --- a/executor/setup.go +++ b/executor/setup.go @@ -4,6 +4,7 @@ package executor import ( "fmt" + "github.com/go-vela/server/storage" "strings" "time" @@ -60,6 +61,8 @@ type Setup struct { Pipeline *pipeline.Build // id token request token for the build RequestToken string + // storage client for interacting with storage resources + Storage *storage.Setup } // Darwin creates and returns a Vela engine capable of @@ -91,6 +94,7 @@ func (s *Setup) Linux() (Engine, error) { linux.WithVersion(s.Version), linux.WithLogger(s.Logger), linux.WithOutputCtn(s.OutputCtn), + linux.WithStorage(s.Storage), ) } @@ -169,6 +173,11 @@ func (s *Setup) Validate() error { return fmt.Errorf("no Vela user provided in setup") } + // check if the storage client is provided + if &s.Storage == nil { + return fmt.Errorf("no storage client provided in setup") + } + // setup is valid return nil } diff --git a/executor/setup_test.go b/executor/setup_test.go index e89700a7..8bcba8c4 100644 --- a/executor/setup_test.go +++ b/executor/setup_test.go @@ -3,6 +3,7 @@ package executor import ( + "github.com/go-vela/server/storage" "net/http/httptest" "testing" "time" @@ -69,6 +70,17 @@ func TestExecutor_Setup_Linux(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + want, err := linux.New( linux.WithBuild(_build), linux.WithMaxLogSize(2097152), @@ -78,6 +90,7 @@ func TestExecutor_Setup_Linux(t *testing.T) { linux.WithRuntime(_runtime), linux.WithVelaClient(_client), linux.WithVersion("v1.0.0"), + linux.WithStorage(_storage), ) if err != nil { t.Errorf("unable to create linux engine: %v", err) @@ -92,6 +105,7 @@ func TestExecutor_Setup_Linux(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", + Storage: _storage, } // run test diff --git a/internal/build/snapshot_test.go b/internal/build/snapshot_test.go index 3baafcba..9658f908 100644 --- a/internal/build/snapshot_test.go +++ b/internal/build/snapshot_test.go @@ -25,7 +25,7 @@ func TestBuild_Snapshot(t *testing.T) { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -40,8 +40,8 @@ func TestBuild_Snapshot(t *testing.T) { b := &api.Build{ ID: vela.Int64(1), Repo: r, - Number: vela.Int(1), - Parent: vela.Int(1), + Number: vela.Int64(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), diff --git a/internal/build/upload_test.go b/internal/build/upload_test.go index 89485e62..41a2e929 100644 --- a/internal/build/upload_test.go +++ b/internal/build/upload_test.go @@ -25,7 +25,7 @@ func TestBuild_Upload(t *testing.T) { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -39,9 +39,9 @@ func TestBuild_Upload(t *testing.T) { _build := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), diff --git a/internal/service/snapshot_test.go b/internal/service/snapshot_test.go index bee7c4ab..b763630b 100644 --- a/internal/service/snapshot_test.go +++ b/internal/service/snapshot_test.go @@ -24,7 +24,7 @@ func TestService_Snapshot(t *testing.T) { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -34,9 +34,9 @@ func TestService_Snapshot(t *testing.T) { _build := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), @@ -87,11 +87,11 @@ func TestService_Snapshot(t *testing.T) { ID: vela.Int64(1), BuildID: vela.Int64(1), RepoID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int32(1), Name: vela.String("postgres"), Image: vela.String("postgres:12-alpine"), Status: vela.String("running"), - ExitCode: vela.Int(0), + ExitCode: vela.Int32(0), Created: vela.Int64(1563474076), Started: vela.Int64(0), Finished: vela.Int64(1563474079), diff --git a/internal/service/upload_test.go b/internal/service/upload_test.go index abedb205..41619bed 100644 --- a/internal/service/upload_test.go +++ b/internal/service/upload_test.go @@ -24,7 +24,7 @@ func TestService_Upload(t *testing.T) { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -34,9 +34,9 @@ func TestService_Upload(t *testing.T) { _build := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), @@ -85,11 +85,11 @@ func TestService_Upload(t *testing.T) { ID: vela.Int64(1), BuildID: vela.Int64(1), RepoID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int32(1), Name: vela.String("postgres"), Image: vela.String("postgres:12-alpine"), Status: vela.String("running"), - ExitCode: vela.Int(0), + ExitCode: vela.Int32(0), Created: vela.Int64(1563474076), Started: vela.Int64(0), Finished: vela.Int64(1563474079), diff --git a/internal/step/skip_test.go b/internal/step/skip_test.go index f422c057..bfb77af3 100644 --- a/internal/step/skip_test.go +++ b/internal/step/skip_test.go @@ -20,7 +20,7 @@ func TestStep_Skip(t *testing.T) { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -30,9 +30,9 @@ func TestStep_Skip(t *testing.T) { _build := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), EventAction: vela.String(""), Status: vela.String("success"), @@ -59,9 +59,9 @@ func TestStep_Skip(t *testing.T) { _comment := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("comment"), EventAction: vela.String("created"), Status: vela.String("success"), @@ -88,9 +88,9 @@ func TestStep_Skip(t *testing.T) { _deploy := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("deployment"), EventAction: vela.String(""), Status: vela.String("success"), @@ -117,9 +117,9 @@ func TestStep_Skip(t *testing.T) { _deployFromTag := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("deployment"), EventAction: vela.String(""), Status: vela.String("success"), @@ -146,9 +146,9 @@ func TestStep_Skip(t *testing.T) { _schedule := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("schedule"), EventAction: vela.String(""), Status: vela.String("success"), @@ -175,9 +175,9 @@ func TestStep_Skip(t *testing.T) { _tag := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("tag"), EventAction: vela.String(""), Status: vela.String("success"), diff --git a/internal/step/snapshot_test.go b/internal/step/snapshot_test.go index 835d238d..348eb6e6 100644 --- a/internal/step/snapshot_test.go +++ b/internal/step/snapshot_test.go @@ -24,7 +24,7 @@ func TestStep_Snapshot(t *testing.T) { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -34,9 +34,9 @@ func TestStep_Snapshot(t *testing.T) { _build := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), @@ -85,11 +85,11 @@ func TestStep_Snapshot(t *testing.T) { ID: vela.Int64(1), BuildID: vela.Int64(1), RepoID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int32(1), Name: vela.String("clone"), Image: vela.String("target/vela-git:v0.3.0"), Status: vela.String("running"), - ExitCode: vela.Int(0), + ExitCode: vela.Int32(0), Created: vela.Int64(1563474076), Started: vela.Int64(0), Finished: vela.Int64(1563474079), @@ -148,7 +148,7 @@ func TestStep_SnapshotInit(t *testing.T) { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -158,9 +158,9 @@ func TestStep_SnapshotInit(t *testing.T) { _build := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), @@ -209,11 +209,11 @@ func TestStep_SnapshotInit(t *testing.T) { ID: vela.Int64(1), BuildID: vela.Int64(1), RepoID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int32(1), Name: vela.String("clone"), Image: vela.String("target/vela-git:v0.3.0"), Status: vela.String("running"), - ExitCode: vela.Int(0), + ExitCode: vela.Int32(0), Created: vela.Int64(1563474076), Started: vela.Int64(0), Finished: vela.Int64(1563474079), diff --git a/runtime/docker/container.go b/runtime/docker/container.go index 811b69ca..c1679264 100644 --- a/runtime/docker/container.go +++ b/runtime/docker/container.go @@ -34,7 +34,7 @@ func (c *client) InspectContainer(ctx context.Context, ctn *pipeline.Container) // capture the container exit code // // https://pkg.go.dev/github.com/docker/docker/api/types#ContainerState - ctn.ExitCode = container.State.ExitCode + ctn.ExitCode = int32(container.State.ExitCode) return nil } diff --git a/runtime/docker/test_report.go b/runtime/docker/test_report.go new file mode 100644 index 00000000..5c5fa55d --- /dev/null +++ b/runtime/docker/test_report.go @@ -0,0 +1,170 @@ +package docker + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "github.com/docker/docker/api/types" + "github.com/docker/docker/pkg/stdcopy" + "github.com/go-vela/server/compiler/types/pipeline" + "io" + "path/filepath" + "strings" +) + +// PollFileNames grabs files name from provided path +// within a container and uploads them to s3 +func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, paths []string) ([]string, error) { + c.Logger.Tracef("gathering test results and attachments from container %s", ctn.ID) + + var fullFilePaths []string + if len(ctn.Image) == 0 { + return nil, nil + } + // iterate through the steps in the build + // iterate through the results paths and store them in the map + for _, path := range paths { + dir, filename := filepath.Split(path) + c.Logger.Tracef("searching for file %s in %s", filename, dir) + + execConfig := types.ExecConfig{ + Tty: true, + //Cmd: []string{"sh", "-c", fmt.Sprintf("find %s -type f -name %s", dir, filename)}, + Cmd: []string{"sh", "-c", fmt.Sprintf("find / -type f -path *%s -print", path)}, + AttachStderr: true, + AttachStdout: true, + } + + c.Logger.Infof("executing command: %v", execConfig.Cmd) + responseExec, err := c.Docker.ContainerExecCreate(ctx, ctn.ID, execConfig) + if err != nil { + c.Logger.Errorf("unable to create exec for container: %v", err) + return nil, err + } + + hijackedResponse, err := c.Docker.ContainerExecAttach(ctx, responseExec.ID, types.ExecStartCheck{}) + if err != nil { + c.Logger.Errorf("unable to attach to exec for container: %v", err) + return nil, err + } + + defer func() { + if hijackedResponse.Conn != nil { + hijackedResponse.Close() + } + }() + + outputStdout := new(bytes.Buffer) + outputStderr := new(bytes.Buffer) + + if hijackedResponse.Reader != nil { + _, err := stdcopy.StdCopy(outputStdout, outputStderr, hijackedResponse.Reader) + if err != nil { + c.Logger.Errorf("unable to copy logs for container: %v", err) + } + } + + if outputStderr.Len() > 0 { + return nil, fmt.Errorf("error: %s", outputStderr.String()) + } + + data := outputStdout.String() + c.Logger.Infof("found files: %s", data) + + filePaths := strings.Split(data, "\n") + for _, filePath := range filePaths { + if filePath != "" { + + fullFilePaths = append(fullFilePaths, strings.TrimSpace(filePath)) + c.Logger.Infof("full file: %s", filePath) + } + } + } + if len(fullFilePaths) == 0 { + return nil, fmt.Errorf("no matching files found for any provided paths") + } + + return fullFilePaths, nil + + // iterate through the steps in the build + //for _, step := range p.Steps { + // if len(step.TestReport.Results) == 0 { + // c.Logger.Warnf("no results provided for the step %s", step.ID) + // return fmt.Errorf("no results provided for the step %s", step.ID) + // } + // if len(step.TestReport.Attachments) == 0 { + // c.Logger.Warnf("no attachments provided for the step %s", step.ID) + // return fmt.Errorf("no attachments provided for the step %s", step.ID) + // } + // // check if the step has the provided paths from results + // for _, result := range step.TestReport.Results { + // _, err := os.Stat(result) + // if err != nil { + // c.Logger.Errorf("unable to find test result %s for step %s", result, step.ID) + // continue + // } + // } + //} + //return nil +} + +// PollFileContent retrieves the content and size of a file inside a container. +func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, path string) (io.Reader, int64, error) { + c.Logger.Tracef("gathering test results and attachments from container %s", ctn.ID) + + if len(ctn.Image) == 0 { + return nil, 0, nil + } + cmd := []string{"sh", "-c", fmt.Sprintf("base64 %s", path)} + execConfig := types.ExecConfig{ + Cmd: cmd, + AttachStdout: true, + AttachStderr: false, + Tty: false, + } + + c.Logger.Infof("executing command for content: %v", execConfig.Cmd) + execID, err := c.Docker.ContainerExecCreate(ctx, ctn.ID, execConfig) + if err != nil { + return nil, 0, fmt.Errorf("failed to create exec instance: %w", err) + } + resp, err := c.Docker.ContainerExecAttach(ctx, execID.ID, types.ExecStartCheck{}) + if err != nil { + return nil, 0, fmt.Errorf("failed to attach to exec instance: %w", err) + } + + defer func() { + if resp.Conn != nil { + resp.Close() + } + }() + + outputStdout := new(bytes.Buffer) + outputStderr := new(bytes.Buffer) + + if resp.Reader != nil { + _, err := stdcopy.StdCopy(outputStdout, outputStderr, resp.Reader) + if err != nil { + c.Logger.Errorf("unable to copy logs for container: %v", err) + } + } + + if outputStderr.Len() > 0 { + return nil, 0, fmt.Errorf("error: %s", outputStderr.String()) + } + data := outputStdout.Bytes() + + decoded, err := base64.StdEncoding.DecodeString(string(data)) + if err != nil { + c.Logger.Errorf("unable to decode base64 data: %v", err) + return nil, 0, err + } + //c.Logger.Infof("data: %v", string(data)) + + // convert the data to a reader + reader := bytes.NewReader(decoded) + // get the size of the data + size := int64(len(decoded)) + return reader, size, nil +} diff --git a/runtime/docker/test_report_test.go b/runtime/docker/test_report_test.go new file mode 100644 index 00000000..84ef352d --- /dev/null +++ b/runtime/docker/test_report_test.go @@ -0,0 +1,76 @@ +package docker + +// +//import ( +// "context" +// "github.com/go-vela/server/compiler/types/pipeline" +// "testing" +//) +// +//func TestClient_TestReport(t *testing.T) { +// // setup client +// _engine, err := NewMock() +// if err != nil { +// t.Errorf("unable to create runtime engine: %v", err) +// } +// +// // setup tests +// tests := []struct { +// name string +// failure bool +// container *pipeline.Container +// }{ +// { +// name: "valid test report", +// failure: false, +// container: &pipeline.Container{ +// ID: "test_container", +// TestReport: pipeline.TestReport{ +// Results: []string{"test_result_path"}, +// Attachments: []string{"test_attachment_path"}, +// }, +// }, +// }, +// { +// name: "no results provided", +// failure: true, +// container: &pipeline.Container{ +// ID: "test_container", +// TestReport: pipeline.TestReport{ +// Results: []string{}, +// Attachments: []string{"test_attachment_path"}, +// }, +// }, +// }, +// { +// name: "no attachments provided", +// failure: true, +// container: &pipeline.Container{ +// ID: "test_container", +// TestReport: pipeline.TestReport{ +// Results: []string{"test_result_path"}, +// Attachments: []string{}, +// }, +// }, +// }, +// } +// +// // run tests +// for _, test := range tests { +// t.Run(test.name, func(t *testing.T) { +// err := _engine.TestReport(context.Background(), test.container) +// +// if test.failure { +// if err == nil { +// t.Errorf("TestReport should have returned err") +// } +// +// return // continue to next test +// } +// +// if err != nil { +// t.Errorf("TestReport returned err: %v", err) +// } +// }) +// } +//} diff --git a/runtime/engine.go b/runtime/engine.go index e842a9c0..f615df06 100644 --- a/runtime/engine.go +++ b/runtime/engine.go @@ -94,4 +94,12 @@ type Engine interface { // RemoveVolume defines a function that // deletes the pipeline volume. RemoveVolume(context.Context, *pipeline.Build) error + // TestReport Interface functions + + // PollFileNames defines a function that + // captures the test results from the pipeline container. + PollFileNames(ctx context.Context, ctn *pipeline.Container, paths []string) ([]string, error) + // PollFileContent defines a function that + // captures the content and size of a file from the pipeline container. + PollFileContent(ctx context.Context, ctn *pipeline.Container, path string) (io.Reader, int64, error) } diff --git a/runtime/kubernetes/container.go b/runtime/kubernetes/container.go index 576cf417..5c78d093 100644 --- a/runtime/kubernetes/container.go +++ b/runtime/kubernetes/container.go @@ -53,7 +53,7 @@ func (c *client) InspectContainer(ctx context.Context, ctn *pipeline.Container) } // set the step exit code - ctn.ExitCode = int(cst.State.Terminated.ExitCode) + ctn.ExitCode = cst.State.Terminated.ExitCode break } @@ -77,6 +77,22 @@ func (c *client) PollOutputsContainer(ctx context.Context, ctn *pipeline.Contain return nil, nil } +// PollFileNames grabs test results and attachments from provided path within a container. +// This is a no-op for kubernetes. Pod environments cannot be dynamic. +func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, paths []string) ([]string, error) { + c.Logger.Tracef("no-op: gathering test results and attachments from container %s", ctn.ID) + + return nil, nil +} + +// PollFileContent captures the content and size of a file from the pipeline container. +// This is a no-op for kubernetes. Pod environments cannot be dynamic. +func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, path string) (io.Reader, int64, error) { + c.Logger.Tracef("no-op: gathering test results and attachments from container %s", ctn.ID) + + return nil, 0, nil +} + // RunContainer creates and starts the pipeline container. func (c *client) RunContainer(ctx context.Context, ctn *pipeline.Container, _ *pipeline.Build) error { c.Logger.Tracef("running container %s", ctn.ID) From 802831e00a89e27ae123495b6a20bf3dd765a853 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Mon, 7 Apr 2025 14:36:15 -0500 Subject: [PATCH 02/63] fix(int): fix int types --- cmd/vela-worker/operate.go | 2 +- executor/executor_test.go | 6 ++-- executor/linux/build.go | 2 +- executor/linux/linux_test.go | 6 ++-- executor/local/local_test.go | 6 ++-- go.mod | 30 ++++++++++------- go.sum | 55 ++++++++++++++++++------------- internal/build/snapshot_test.go | 6 ++-- internal/build/upload_test.go | 6 ++-- internal/service/snapshot_test.go | 10 +++--- internal/service/upload_test.go | 10 +++--- internal/step/skip_test.go | 26 +++++++-------- internal/step/snapshot_test.go | 20 +++++------ internal/step/upload_test.go | 10 +++--- runtime/docker/container.go | 2 +- runtime/kubernetes/container.go | 2 +- 16 files changed, 107 insertions(+), 92 deletions(-) diff --git a/cmd/vela-worker/operate.go b/cmd/vela-worker/operate.go index 46c2bc34..f517b523 100644 --- a/cmd/vela-worker/operate.go +++ b/cmd/vela-worker/operate.go @@ -31,7 +31,7 @@ func (w *Worker) operate(ctx context.Context) error { registryWorker.SetHostname(w.Config.API.Address.Hostname()) registryWorker.SetAddress(w.Config.API.Address.String()) registryWorker.SetActive(true) - registryWorker.SetBuildLimit(int64(w.Config.Build.Limit)) + registryWorker.SetBuildLimit(int32(w.Config.Build.Limit)) // set routes from config if set or defaulted to `vela` if (len(w.Config.Queue.Routes) > 0) && (w.Config.Queue.Routes[0] != "NONE" && w.Config.Queue.Routes[0] != "") { diff --git a/executor/executor_test.go b/executor/executor_test.go index 8282d75f..48b0c6a5 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -260,7 +260,7 @@ var ( Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -271,9 +271,9 @@ var ( _build = &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), diff --git a/executor/linux/build.go b/executor/linux/build.go index 437b225b..60d660ed 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -396,7 +396,7 @@ func (c *client) AssembleBuild(ctx context.Context) error { } // create outputs container with a timeout equal to the repo timeout - c.err = c.outputs.create(ctx, c.OutputCtn, (int64(60) * c.build.GetRepo().GetTimeout())) + c.err = c.outputs.create(ctx, c.OutputCtn, int64(60*c.build.GetRepo().GetTimeout())) if c.err != nil { return fmt.Errorf("unable to create outputs container: %w", c.err) } diff --git a/executor/linux/linux_test.go b/executor/linux/linux_test.go index d6c7d0fd..9a026c87 100644 --- a/executor/linux/linux_test.go +++ b/executor/linux/linux_test.go @@ -168,9 +168,9 @@ func TestLinux_New(t *testing.T) { func testBuild() *api.Build { return &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: testRepo(), - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), @@ -206,7 +206,7 @@ func testRepo() *api.Repo { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), diff --git a/executor/local/local_test.go b/executor/local/local_test.go index 26c7c9cc..4479920b 100644 --- a/executor/local/local_test.go +++ b/executor/local/local_test.go @@ -164,9 +164,9 @@ func TestLocal_New(t *testing.T) { func testBuild() *api.Build { return &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: testRepo(), - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), @@ -202,7 +202,7 @@ func testRepo() *api.Repo { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), diff --git a/go.mod b/go.mod index ccaa854d..0c32345b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/go-vela/worker -go 1.23.5 +go 1.24.2 + require ( github.com/Masterminds/semver/v3 v3.3.1 @@ -9,7 +10,7 @@ require ( github.com/docker/go-units v0.5.0 github.com/gin-gonic/gin v1.10.0 github.com/go-vela/sdk-go v0.26.2 - github.com/go-vela/server v0.26.3 + github.com/go-vela/server v0.26.4 github.com/golang-jwt/jwt/v5 v5.2.2 github.com/google/go-cmp v0.7.0 github.com/joho/godotenv v1.5.1 @@ -20,7 +21,7 @@ require ( golang.org/x/sync v0.12.0 gotest.tools/v3 v3.5.2 k8s.io/api v0.32.2 - k8s.io/apimachinery v0.32.2 + k8s.io/apimachinery v0.32.3 k8s.io/client-go v0.32.2 sigs.k8s.io/yaml v1.4.0 ) @@ -32,8 +33,10 @@ require ( github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/google/go-github/v69 v69.2.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/google/go-github/v70 v70.0.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 // indirect github.com/invopop/jsonschema v0.13.0 // indirect @@ -43,10 +46,16 @@ require ( github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/lestrrat-go/httprc/v3 v3.0.0-beta1 // indirect + github.com/lestrrat-go/jwx/v3 v3.0.0 // indirect github.com/lib/pq v1.10.9 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect + github.com/minio/crc64nvme v1.0.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.89 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/uptrace/opentelemetry-go-extra/otelgorm v0.3.2 // indirect github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect @@ -122,9 +131,6 @@ require ( github.com/leodido/go-urn v1.4.0 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect - github.com/lestrrat-go/httprc v1.0.6 // indirect - github.com/lestrrat-go/iter v1.0.2 // indirect - github.com/lestrrat-go/jwx/v2 v2.1.4 // indirect github.com/lestrrat-go/option v1.0.1 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -142,11 +148,11 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/redis/go-redis/v9 v9.7.1 // indirect + github.com/redis/go-redis/v9 v9.7.3 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect - github.com/spf13/afero v1.12.0 // indirect + github.com/spf13/afero v1.14.0 // indirect github.com/spf13/cast v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect @@ -157,11 +163,11 @@ require ( go.opentelemetry.io/otel v1.35.0 // indirect go.opentelemetry.io/otel/metric v1.35.0 // indirect go.opentelemetry.io/otel/trace v1.35.0 // indirect - go.starlark.net v0.0.0-20250225190231-0d3f41d403af // indirect + go.starlark.net v0.0.0-20250318223901-d9371fef63fe // indirect golang.org/x/arch v0.14.0 // indirect golang.org/x/crypto v0.36.0 // indirect golang.org/x/mod v0.21.0 // indirect - golang.org/x/net v0.35.0 // indirect + golang.org/x/net v0.37.0 // indirect golang.org/x/oauth2 v0.28.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/term v0.30.0 // indirect diff --git a/go.sum b/go.sum index 851450d8..5ad407c1 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= @@ -96,6 +98,8 @@ github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -119,10 +123,8 @@ github.com/go-playground/validator/v10 v10.25.0 h1:5Dh7cjvzR7BRZadnsVOzPhWsrwUr0 github.com/go-playground/validator/v10 v10.25.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/go-vela/sdk-go v0.26.2 h1:uqYZNO9VYfDiyXl7MMgUt6XwBwvIj3OWWAIa8q9RuXE= github.com/go-vela/sdk-go v0.26.2/go.mod h1:XLnki5qdk8R+y+J65AOuiJ1pQyBmy7P46E85SPOv0Ww= -github.com/go-vela/server v0.26.3 h1:2BvXMyfzyZq4tgNaq/AiHGYNr0FJKEiWtXDeas4B7JI= -github.com/go-vela/server v0.26.3/go.mod h1:SNQVpgPoCmd7dvSA+nILmsFF0Ijoh0nEm24ytJd9J4o= +github.com/go-vela/server v0.26.4/go.mod h1:SNQVpgPoCmd7dvSA+nILmsFF0Ijoh0nEm24ytJd9J4o= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -140,8 +142,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-github/v69 v69.2.0 h1:wR+Wi/fN2zdUx9YxSmYE0ktiX9IAR/BeePzeaUUbEHE= -github.com/google/go-github/v69 v69.2.0/go.mod h1:xne4jymxLR6Uj9b7J7PyTpkMYstEMMwGZa0Aehh1azM= +github.com/google/go-github/v70 v70.0.0 h1:/tqCp5KPrcvqCc7vIvYyFYTiCGrYvaWoYMGHSQbo55o= +github.com/google/go-github/v70 v70.0.0/go.mod h1:xBUZgo8MI3lUL/hwxl3hlceJW1U8MVnXP3zUyI+rhQY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -194,8 +196,9 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= @@ -215,12 +218,10 @@ github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= -github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCGW8k= -github.com/lestrrat-go/httprc v1.0.6/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= -github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= -github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= -github.com/lestrrat-go/jwx/v2 v2.1.4 h1:uBCMmJX8oRZStmKuMMOFb0Yh9xmEMgNJLgjuKKt4/qc= -github.com/lestrrat-go/jwx/v2 v2.1.4/go.mod h1:nWRbDFR1ALG2Z6GJbBXzfQaYyvn751KuuyySN2yR6is= +github.com/lestrrat-go/httprc/v3 v3.0.0-beta1 h1:pzDjP9dSONCFQC/AE3mWUnHILGiYPiMKzQIS+weKJXA= +github.com/lestrrat-go/httprc/v3 v3.0.0-beta1/go.mod h1:wdsgouffPvWPEYh8t7PRH/PidR5sfVqt0na4Nhj60Ms= +github.com/lestrrat-go/jwx/v3 v3.0.0 h1:IRnFNdZx5dJHjTpPVkYqP6TRahJI2Z9v43UwEDJcj6U= +github.com/lestrrat-go/jwx/v3 v3.0.0/go.mod h1:ak32WoNtHE0aLowVWBcCvXngcAnW4tuC0YhFwOr/kwc= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -235,6 +236,12 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY= +github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.89 h1:hx4xV5wwTUfyv8LarhJAwNecnXpoTsj9v3f3q/ZkiJU= +github.com/minio/minio-go/v7 v7.0.89/go.mod h1:2rFnGAp02p7Dddo1Fq4S2wYOfpF0MUTSeLTRC90I204= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -276,10 +283,12 @@ github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/redis/go-redis/v9 v9.7.1 h1:4LhKRCIduqXqtvCUlaq9c8bdHOkICjDMrr1+Zb3osAc= -github.com/redis/go-redis/v9 v9.7.1/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= +github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= +github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= @@ -288,8 +297,8 @@ github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= -github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= +github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -350,8 +359,8 @@ go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= -go.starlark.net v0.0.0-20250225190231-0d3f41d403af h1:gdHSl5pZSdC+7qdBKx0n0x4Y2b4UNjuKnKH8Lfwft3o= -go.starlark.net v0.0.0-20250225190231-0d3f41d403af/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8= +go.starlark.net v0.0.0-20250318223901-d9371fef63fe h1:Wf00k2WTLCW/L1/+gA1gxfTcU4yI+nK4YRTjumYezD8= +go.starlark.net v0.0.0-20250318223901-d9371fef63fe/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/arch v0.14.0 h1:z9JUEZWr8x4rR0OU6c4/4t6E6jOZ8/QBS2bBYBm4tx4= @@ -371,8 +380,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= -golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= +golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= +golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -438,8 +447,8 @@ gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= k8s.io/api v0.32.2 h1:bZrMLEkgizC24G9eViHGOPbW+aRo9duEISRIJKfdJuw= k8s.io/api v0.32.2/go.mod h1:hKlhk4x1sJyYnHENsrdCWw31FEmCijNGPJO5WzHiJ6Y= -k8s.io/apimachinery v0.32.2 h1:yoQBR9ZGkA6Rgmhbp/yuT9/g+4lxtsGYwW6dR6BDPLQ= -k8s.io/apimachinery v0.32.2/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U= +k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= k8s.io/client-go v0.32.2 h1:4dYCD4Nz+9RApM2b/3BtVvBHw54QjMFUl1OLcJG5yOA= k8s.io/client-go v0.32.2/go.mod h1:fpZ4oJXclZ3r2nDOv+Ux3XcJutfrwjKTCHz2H3sww94= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= diff --git a/internal/build/snapshot_test.go b/internal/build/snapshot_test.go index 3baafcba..9658f908 100644 --- a/internal/build/snapshot_test.go +++ b/internal/build/snapshot_test.go @@ -25,7 +25,7 @@ func TestBuild_Snapshot(t *testing.T) { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -40,8 +40,8 @@ func TestBuild_Snapshot(t *testing.T) { b := &api.Build{ ID: vela.Int64(1), Repo: r, - Number: vela.Int(1), - Parent: vela.Int(1), + Number: vela.Int64(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), diff --git a/internal/build/upload_test.go b/internal/build/upload_test.go index 89485e62..41a2e929 100644 --- a/internal/build/upload_test.go +++ b/internal/build/upload_test.go @@ -25,7 +25,7 @@ func TestBuild_Upload(t *testing.T) { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -39,9 +39,9 @@ func TestBuild_Upload(t *testing.T) { _build := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), diff --git a/internal/service/snapshot_test.go b/internal/service/snapshot_test.go index bee7c4ab..b763630b 100644 --- a/internal/service/snapshot_test.go +++ b/internal/service/snapshot_test.go @@ -24,7 +24,7 @@ func TestService_Snapshot(t *testing.T) { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -34,9 +34,9 @@ func TestService_Snapshot(t *testing.T) { _build := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), @@ -87,11 +87,11 @@ func TestService_Snapshot(t *testing.T) { ID: vela.Int64(1), BuildID: vela.Int64(1), RepoID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int32(1), Name: vela.String("postgres"), Image: vela.String("postgres:12-alpine"), Status: vela.String("running"), - ExitCode: vela.Int(0), + ExitCode: vela.Int32(0), Created: vela.Int64(1563474076), Started: vela.Int64(0), Finished: vela.Int64(1563474079), diff --git a/internal/service/upload_test.go b/internal/service/upload_test.go index abedb205..41619bed 100644 --- a/internal/service/upload_test.go +++ b/internal/service/upload_test.go @@ -24,7 +24,7 @@ func TestService_Upload(t *testing.T) { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -34,9 +34,9 @@ func TestService_Upload(t *testing.T) { _build := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), @@ -85,11 +85,11 @@ func TestService_Upload(t *testing.T) { ID: vela.Int64(1), BuildID: vela.Int64(1), RepoID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int32(1), Name: vela.String("postgres"), Image: vela.String("postgres:12-alpine"), Status: vela.String("running"), - ExitCode: vela.Int(0), + ExitCode: vela.Int32(0), Created: vela.Int64(1563474076), Started: vela.Int64(0), Finished: vela.Int64(1563474079), diff --git a/internal/step/skip_test.go b/internal/step/skip_test.go index f422c057..bfb77af3 100644 --- a/internal/step/skip_test.go +++ b/internal/step/skip_test.go @@ -20,7 +20,7 @@ func TestStep_Skip(t *testing.T) { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -30,9 +30,9 @@ func TestStep_Skip(t *testing.T) { _build := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), EventAction: vela.String(""), Status: vela.String("success"), @@ -59,9 +59,9 @@ func TestStep_Skip(t *testing.T) { _comment := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("comment"), EventAction: vela.String("created"), Status: vela.String("success"), @@ -88,9 +88,9 @@ func TestStep_Skip(t *testing.T) { _deploy := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("deployment"), EventAction: vela.String(""), Status: vela.String("success"), @@ -117,9 +117,9 @@ func TestStep_Skip(t *testing.T) { _deployFromTag := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("deployment"), EventAction: vela.String(""), Status: vela.String("success"), @@ -146,9 +146,9 @@ func TestStep_Skip(t *testing.T) { _schedule := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("schedule"), EventAction: vela.String(""), Status: vela.String("success"), @@ -175,9 +175,9 @@ func TestStep_Skip(t *testing.T) { _tag := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("tag"), EventAction: vela.String(""), Status: vela.String("success"), diff --git a/internal/step/snapshot_test.go b/internal/step/snapshot_test.go index 835d238d..348eb6e6 100644 --- a/internal/step/snapshot_test.go +++ b/internal/step/snapshot_test.go @@ -24,7 +24,7 @@ func TestStep_Snapshot(t *testing.T) { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -34,9 +34,9 @@ func TestStep_Snapshot(t *testing.T) { _build := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), @@ -85,11 +85,11 @@ func TestStep_Snapshot(t *testing.T) { ID: vela.Int64(1), BuildID: vela.Int64(1), RepoID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int32(1), Name: vela.String("clone"), Image: vela.String("target/vela-git:v0.3.0"), Status: vela.String("running"), - ExitCode: vela.Int(0), + ExitCode: vela.Int32(0), Created: vela.Int64(1563474076), Started: vela.Int64(0), Finished: vela.Int64(1563474079), @@ -148,7 +148,7 @@ func TestStep_SnapshotInit(t *testing.T) { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -158,9 +158,9 @@ func TestStep_SnapshotInit(t *testing.T) { _build := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), @@ -209,11 +209,11 @@ func TestStep_SnapshotInit(t *testing.T) { ID: vela.Int64(1), BuildID: vela.Int64(1), RepoID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int32(1), Name: vela.String("clone"), Image: vela.String("target/vela-git:v0.3.0"), Status: vela.String("running"), - ExitCode: vela.Int(0), + ExitCode: vela.Int32(0), Created: vela.Int64(1563474076), Started: vela.Int64(0), Finished: vela.Int64(1563474079), diff --git a/internal/step/upload_test.go b/internal/step/upload_test.go index 026c10b4..5d527f19 100644 --- a/internal/step/upload_test.go +++ b/internal/step/upload_test.go @@ -24,7 +24,7 @@ func TestStep_Upload(t *testing.T) { Link: vela.String("https://github.com/github/octocat"), Clone: vela.String("https://github.com/github/octocat.git"), Branch: vela.String("main"), - Timeout: vela.Int64(60), + Timeout: vela.Int32(60), Visibility: vela.String("public"), Private: vela.Bool(false), Trusted: vela.Bool(false), @@ -34,9 +34,9 @@ func TestStep_Upload(t *testing.T) { _build := &api.Build{ ID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int64(1), Repo: _repo, - Parent: vela.Int(1), + Parent: vela.Int64(1), Event: vela.String("push"), Status: vela.String("success"), Error: vela.String(""), @@ -85,11 +85,11 @@ func TestStep_Upload(t *testing.T) { ID: vela.Int64(1), BuildID: vela.Int64(1), RepoID: vela.Int64(1), - Number: vela.Int(1), + Number: vela.Int32(1), Name: vela.String("clone"), Image: vela.String("target/vela-git:v0.3.0"), Status: vela.String("running"), - ExitCode: vela.Int(0), + ExitCode: vela.Int32(0), Created: vela.Int64(1563474076), Started: vela.Int64(0), Finished: vela.Int64(1563474079), diff --git a/runtime/docker/container.go b/runtime/docker/container.go index 811b69ca..c1679264 100644 --- a/runtime/docker/container.go +++ b/runtime/docker/container.go @@ -34,7 +34,7 @@ func (c *client) InspectContainer(ctx context.Context, ctn *pipeline.Container) // capture the container exit code // // https://pkg.go.dev/github.com/docker/docker/api/types#ContainerState - ctn.ExitCode = container.State.ExitCode + ctn.ExitCode = int32(container.State.ExitCode) return nil } diff --git a/runtime/kubernetes/container.go b/runtime/kubernetes/container.go index 576cf417..1512a8da 100644 --- a/runtime/kubernetes/container.go +++ b/runtime/kubernetes/container.go @@ -53,7 +53,7 @@ func (c *client) InspectContainer(ctx context.Context, ctn *pipeline.Container) } // set the step exit code - ctn.ExitCode = int(cst.State.Terminated.ExitCode) + ctn.ExitCode = cst.State.Terminated.ExitCode break } From 309d53ace23edbaa753944836571ed4de3e8b79b Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Tue, 8 Apr 2025 08:44:17 -0500 Subject: [PATCH 03/63] changes --- cmd/vela-worker/run.go | 2 +- cmd/vela-worker/worker.go | 2 +- executor/setup.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/vela-worker/run.go b/cmd/vela-worker/run.go index 6b397b20..dbe5e287 100644 --- a/cmd/vela-worker/run.go +++ b/cmd/vela-worker/run.go @@ -95,7 +95,7 @@ func run(c *cli.Context) error { }, // build configuration Build: &Build{ - Limit: c.Int64("build.limit"), + Limit: c.Int("build.limit"), Timeout: c.Duration("build.timeout"), }, // build configuration diff --git a/cmd/vela-worker/worker.go b/cmd/vela-worker/worker.go index 85ac6d5b..834a88b0 100644 --- a/cmd/vela-worker/worker.go +++ b/cmd/vela-worker/worker.go @@ -22,7 +22,7 @@ type ( // Build represents the worker configuration for build information. Build struct { - Limit int64 + Limit int Timeout time.Duration } diff --git a/executor/setup.go b/executor/setup.go index ef1f1280..f28e95fa 100644 --- a/executor/setup.go +++ b/executor/setup.go @@ -177,7 +177,7 @@ func (s *Setup) Validate() error { if &s.Storage == nil { return fmt.Errorf("no storage client provided in setup") } - + // setup is valid return nil } From aeb5c302b326bd7133a4acb583f22cd894bd6390 Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Tue, 8 Apr 2025 11:27:48 -0500 Subject: [PATCH 04/63] chore(urfave cli): bump to v3 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ccaa854d..190071a2 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/opencontainers/image-spec v1.1.1 github.com/prometheus/client_golang v1.21.1 github.com/sirupsen/logrus v1.9.3 - github.com/urfave/cli/v2 v2.27.6 + github.com/urfave/cli/v3 v3.1.1 golang.org/x/sync v0.12.0 gotest.tools/v3 v3.5.2 k8s.io/api v0.32.2 From e2749c25a6600c5166b1dc612186644bdda620f9 Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Tue, 8 Apr 2025 14:40:47 -0500 Subject: [PATCH 05/63] converting some of the flags --- cmd/vela-worker/flags.go | 40 ++++++++++++----- cmd/vela-worker/main.go | 33 ++++---------- executor/flags.go | 52 +++++++++++++--------- go.mod | 1 + go.sum | 2 + runtime/flags.go | 94 +++++++++++++++++++++++++--------------- 6 files changed, 131 insertions(+), 91 deletions(-) diff --git a/cmd/vela-worker/flags.go b/cmd/vela-worker/flags.go index 4586ea62..8f94d8f7 100644 --- a/cmd/vela-worker/flags.go +++ b/cmd/vela-worker/flags.go @@ -3,9 +3,12 @@ package main import ( + "context" + "fmt" + "strings" "time" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" "github.com/go-vela/server/queue" "github.com/go-vela/worker/executor" @@ -19,43 +22,56 @@ func flags() []cli.Flag { f := []cli.Flag{ &cli.StringFlag{ - EnvVars: []string{"WORKER_ADDR", "VELA_WORKER_ADDR", "VELA_WORKER"}, Name: "worker.addr", Usage: "Worker server address as a fully qualified url (://)", + Sources: cli.EnvVars("WORKER_ADDR", "VELA_WORKER_ADDR", "VELA_WORKER"), + Action: func(_ context.Context, _ *cli.Command, v string) error { + // check if the worker address has a scheme + if !strings.Contains(v, "://") { + return fmt.Errorf("worker address must be fully qualified (://)") + } + + // check if the worker address has a trailing slash + if strings.HasSuffix(v, "/") { + return fmt.Errorf("worker address must not have trailing slash") + } + + return nil + }, }, &cli.DurationFlag{ - EnvVars: []string{"WORKER_CHECK_IN", "VELA_CHECK_IN", "CHECK_IN"}, Name: "checkIn", Usage: "time to wait in between checking in with the server", + Sources: cli.EnvVars("WORKER_CHECK_IN", "VELA_CHECK_IN", "CHECK_IN"), Value: 15 * time.Minute, }, // Build Flags &cli.IntFlag{ - EnvVars: []string{"WORKER_BUILD_LIMIT", "VELA_BUILD_LIMIT", "BUILD_LIMIT"}, Name: "build.limit", Usage: "maximum amount of builds that can run concurrently", + Sources: cli.EnvVars("WORKER_BUILD_LIMIT", "VELA_BUILD_LIMIT", "BUILD_LIMIT"), Value: 1, }, &cli.DurationFlag{ - EnvVars: []string{"WORKER_BUILD_TIMEOUT", "VELA_BUILD_TIMEOUT", "BUILD_TIMEOUT"}, Name: "build.timeout", Usage: "maximum amount of time a build can run for", + Sources: cli.EnvVars("WORKER_BUILD_TIMEOUT", "VELA_BUILD_TIMEOUT", "BUILD_TIMEOUT"), Value: 30 * time.Minute, }, // Logger Flags &cli.StringFlag{ - EnvVars: []string{"WORKER_LOG_FORMAT", "VELA_LOG_FORMAT", "LOG_FORMAT"}, Name: "log.format", Usage: "set log format for the worker", + Sources: cli.EnvVars("WORKER_LOG_FORMAT", "VELA_LOG_FORMAT", "LOG_FORMAT"), Value: "json", }, &cli.StringFlag{ - EnvVars: []string{"WORKER_LOG_LEVEL", "VELA_LOG_LEVEL", "LOG_LEVEL"}, + Sources: cli.EnvVars("WORKER_LOG_LEVEL", "VELA_LOG_LEVEL", "LOG_LEVEL"), Name: "log.level", Usage: "set log level for the worker", Value: "info", @@ -64,30 +80,30 @@ func flags() []cli.Flag { // Server Flags &cli.StringFlag{ - EnvVars: []string{"WORKER_SERVER_ADDR", "VELA_SERVER_ADDR", "VELA_SERVER", "SERVER_ADDR"}, Name: "server.addr", Usage: "Vela server address as a fully qualified url (://)", + Sources: cli.EnvVars("WORKER_SERVER_ADDR", "VELA_SERVER_ADDR", "VELA_SERVER", "SERVER_ADDR"), }, &cli.StringFlag{ - EnvVars: []string{"WORKER_SERVER_SECRET", "VELA_SERVER_SECRET", "SERVER_SECRET"}, Name: "server.secret", Usage: "secret used for server <-> worker communication", + Sources: cli.EnvVars("WORKER_SERVER_SECRET", "VELA_SERVER_SECRET", "SERVER_SECRET"), Value: "", }, &cli.StringFlag{ - EnvVars: []string{"WORKER_SERVER_CERT", "VELA_SERVER_CERT", "SERVER_CERT"}, Name: "server.cert", Usage: "optional TLS certificate for https", + Sources: cli.EnvVars("WORKER_SERVER_CERT", "VELA_SERVER_CERT", "SERVER_CERT"), }, &cli.StringFlag{ - EnvVars: []string{"WORKER_SERVER_CERT_KEY", "VELA_SERVER_CERT_KEY", "SERVER_CERT_KEY"}, Name: "server.cert-key", Usage: "optional TLS certificate key", + Sources: cli.EnvVars("WORKER_SERVER_CERT_KEY", "VELA_SERVER_CERT_KEY", "SERVER_CERT_KEY"), }, &cli.StringFlag{ - EnvVars: []string{"WORKER_SERVER_TLS_MIN_VERSION", "VELA_SERVER_TLS_MIN_VERSION", "SERVER_TLS_MIN_VERSION"}, Name: "server.tls-min-version", Usage: "optional TLS minimum version requirement", + Sources: cli.EnvVars("WORKER_SERVER_TLS_MIN_VERSION", "VELA_SERVER_TLS_MIN_VERSION", "SERVER_TLS_MIN_VERSION"), Value: "1.2", }, } diff --git a/cmd/vela-worker/main.go b/cmd/vela-worker/main.go index 020fecc5..5daf47b1 100644 --- a/cmd/vela-worker/main.go +++ b/cmd/vela-worker/main.go @@ -3,13 +3,13 @@ package main import ( + "context" "encoding/json" "fmt" "os" - "time" "github.com/sirupsen/logrus" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" _ "github.com/joho/godotenv/autoload" @@ -45,35 +45,20 @@ func main() { // output the version information to stdout fmt.Fprintf(os.Stdout, "%s\n", string(bytes)) - app := cli.NewApp() - - // Worker Information - - app.Name = "vela-worker" - app.HelpName = "vela-worker" - app.Usage = "Vela build daemon designed for executing pipelines" - app.Copyright = "Copyright 2019 Target Brands, Inc. All rights reserved." - app.Authors = []*cli.Author{ - { - Name: "Vela Admins", - Email: "vela@target.com", - }, + cmd := cli.Command{ + Name: "vela-worker", + Version: v.Semantic(), + Action: run, + Usage: "Vela build daemon designed for executing pipelines", } - // Worker Metadata - - app.Action = run - app.Compiled = time.Now() - app.Version = v.Semantic() - // Worker Flags - app.Flags = flags() + cmd.Flags = flags() // Worker Start - err = app.Run(os.Args) - if err != nil { + if err = cmd.Run(context.Background(), os.Args); err != nil { logrus.Fatal(err) } } diff --git a/executor/flags.go b/executor/flags.go index e913ab87..c3d53cf1 100644 --- a/executor/flags.go +++ b/executor/flags.go @@ -5,7 +5,7 @@ package executor import ( "time" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" "github.com/go-vela/server/constants" ) @@ -18,35 +18,47 @@ var Flags = []cli.Flag{ // Executor Flags &cli.StringFlag{ - EnvVars: []string{"VELA_EXECUTOR_DRIVER", "EXECUTOR_DRIVER"}, - FilePath: "/vela/executor/driver", - Name: "executor.driver", - Usage: "driver to be used for the executor", - Value: constants.DriverLinux, + Name: "executor.driver", + Usage: "driver to be used for the executor", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_EXECUTOR_DRIVER"), + cli.EnvVar("EXECUTOR_DRIVER"), + cli.File("/vela/executor/driver"), + ), + Value: constants.DriverLinux, }, &cli.UintFlag{ - EnvVars: []string{"VELA_EXECUTOR_MAX_LOG_SIZE", "EXECUTOR_MAX_LOG_SIZE"}, - FilePath: "/vela/executor/max_log_size", - Name: "executor.max_log_size", - Usage: "maximum log size (in bytes)", + Name: "executor.max_log_size", + Usage: "maximum log size (in bytes)", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_EXECUTOR_MAX_LOG_SIZE"), + cli.EnvVar("EXECUTOR_MAX_LOG_SIZE"), + cli.File("/vela/executor/max_log_size"), + ), }, &cli.DurationFlag{ - EnvVars: []string{"WORKER_LOG_STREAMING_TIMEOUT", "VELA_LOG_STREAMING_TIMEOUT", "LOG_STREAMING_TIMEOUT"}, Name: "executor.log_streaming_timeout", Usage: "maximum amount of time to wait for log streaming after build completes", + Sources: cli.EnvVars("WORKER_LOG_STREAMING_TIMEOUT", "VELA_LOG_STREAMING_TIMEOUT", "LOG_STREAMING_TIMEOUT"), Value: 5 * time.Minute, }, &cli.BoolFlag{ - EnvVars: []string{"VELA_EXECUTOR_ENFORCE_TRUSTED_REPOS", "EXECUTOR_ENFORCE_TRUSTED_REPOS"}, - FilePath: "/vela/executor/enforce_trusted_repos", - Name: "executor.enforce-trusted-repos", - Usage: "enforce trusted repo restrictions for privileged images", - Value: true, + Name: "executor.enforce-trusted-repos", + Usage: "enforce trusted repo restrictions for privileged images", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_EXECUTOR_ENFORCE_TRUSTED_REPOS"), + cli.EnvVar("EXECUTOR_ENFORCE_TRUSTED_REPOS"), + cli.File("/vela/executor/enforce_trusted_repos"), + ), + Value: true, }, &cli.StringFlag{ - EnvVars: []string{"VELA_EXECUTOR_OUTPUTS_IMAGE", "EXECUTOR_OUTPUTS_IMAGE"}, - FilePath: "/vela/executor/outputs_image", - Name: "executor.outputs-image", - Usage: "image used for the outputs container sidecar", + Name: "executor.outputs-image", + Usage: "image used for the outputs container sidecar", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_EXECUTOR_OUTPUTS_IMAGE"), + cli.EnvVar("EXECUTOR_OUTPUTS_IMAGE"), + cli.File("/vela/executor/outputs_image"), + ), }, } diff --git a/go.mod b/go.mod index 190071a2..efb8dfd7 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/opencontainers/image-spec v1.1.1 github.com/prometheus/client_golang v1.21.1 github.com/sirupsen/logrus v1.9.3 + github.com/urfave/cli/v2 v2.27.6 github.com/urfave/cli/v3 v3.1.1 golang.org/x/sync v0.12.0 gotest.tools/v3 v3.5.2 diff --git a/go.sum b/go.sum index 851450d8..7ecd0241 100644 --- a/go.sum +++ b/go.sum @@ -318,6 +318,8 @@ github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 h1:ZjUj9BLYf9PEqBn8W/Oa github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2/go.mod h1:O8bHQfyinKwTXKkiKNGmLQS7vRsqRxIQTFZpYpHK3IQ= github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g= github.com/urfave/cli/v2 v2.27.6/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= +github.com/urfave/cli/v3 v3.1.1 h1:bNnl8pFI5dxPOjeONvFCDFoECLQsceDG4ejahs4Jtxk= +github.com/urfave/cli/v3 v3.1.1/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZtpYpo= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= diff --git a/runtime/flags.go b/runtime/flags.go index ded1ebc7..6dbe85d4 100644 --- a/runtime/flags.go +++ b/runtime/flags.go @@ -3,7 +3,7 @@ package runtime import ( - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" "github.com/go-vela/server/constants" ) @@ -16,52 +16,76 @@ var Flags = []cli.Flag{ // Runtime Flags &cli.StringFlag{ - EnvVars: []string{"VELA_RUNTIME_DRIVER", "RUNTIME_DRIVER"}, - FilePath: "/vela/runtime/driver", - Name: "runtime.driver", - Usage: "driver to be used for the runtime", - Value: constants.DriverDocker, + Name: "runtime.driver", + Usage: "driver to be used for the runtime", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_RUNTIME_DRIVER"), + cli.EnvVar("RUNTIME_DRIVER"), + cli.File("/vela/runtime/driver"), + ), + Value: constants.DriverDocker, }, &cli.StringFlag{ - EnvVars: []string{"VELA_RUNTIME_CONFIG", "RUNTIME_CONFIG"}, - FilePath: "/vela/runtime/config", - Name: "runtime.config", - Usage: "path to configuration file for the runtime", + Name: "runtime.config", + Usage: "path to configuration file for the runtime", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_RUNTIME_CONFIG"), + cli.EnvVar("RUNTIME_CONFIG"), + cli.File("/vela/runtime/config"), + ), }, &cli.StringFlag{ - EnvVars: []string{"VELA_RUNTIME_NAMESPACE", "RUNTIME_NAMESPACE"}, - FilePath: "/vela/runtime/namespace", - Name: "runtime.namespace", - Usage: "namespace to use for the runtime (only used by kubernetes)", + Name: "runtime.namespace", + Usage: "namespace to use for the runtime (only used by kubernetes)", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_RUNTIME_NAMESPACE"), + cli.EnvVar("RUNTIME_NAMESPACE"), + cli.File("/vela/runtime/namespace"), + ), }, &cli.StringFlag{ - EnvVars: []string{"VELA_RUNTIME_PODS_TEMPLATE_NAME", "RUNTIME_PODS_TEMPLATE_NAME"}, - FilePath: "/vela/runtime/pods_template_name", - Name: "runtime.pods-template-name", - Usage: "name of the PipelinePodsTemplate to retrieve from the runtime.namespace (only used by kubernetes)", + Name: "runtime.pods-template-name", + Usage: "name of the PipelinePodsTemplate to retrieve from the runtime.namespace (only used by kubernetes)", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_RUNTIME_PODS_TEMPLATE_NAME"), + cli.EnvVar("RUNTIME_PODS_TEMPLATE_NAME"), + cli.File("/vela/runtime/pods_template_name"), + ), }, - &cli.PathFlag{ - EnvVars: []string{"VELA_RUNTIME_PODS_TEMPLATE_FILE", "RUNTIME_PODS_TEMPLATE_FILE"}, - FilePath: "/vela/runtime/pods_template_file", - Name: "runtime.pods-template-file", - Usage: "path to local fallback file containing a PipelinePodsTemplate in YAML (only used by kubernetes; only used if runtime.pods-template-name is not defined)", + &cli.StringFlag{ + Name: "runtime.pods-template-file", + Usage: "path to local fallback file containing a PipelinePodsTemplate in YAML (only used by kubernetes; only used if runtime.pods-template-name is not defined)", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_RUNTIME_PODS_TEMPLATE_FILE"), + cli.EnvVar("RUNTIME_PODS_TEMPLATE_FILE"), + cli.File("/vela/runtime/pods_template_file"), + ), }, &cli.StringSliceFlag{ - EnvVars: []string{"VELA_RUNTIME_PRIVILEGED_IMAGES", "RUNTIME_PRIVILEGED_IMAGES"}, - FilePath: "/vela/runtime/privileged_images", - Name: "runtime.privileged-images", - Usage: "list of images allowed to run in privileged mode for the runtime", + Name: "runtime.privileged-images", + Usage: "list of images allowed to run in privileged mode for the runtime", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_RUNTIME_PRIVILEGED_IMAGES"), + cli.EnvVar("RUNTIME_PRIVILEGED_IMAGES"), + cli.File("/vela/runtime/privileged_images"), + ), }, &cli.StringSliceFlag{ - EnvVars: []string{"VELA_RUNTIME_VOLUMES", "RUNTIME_VOLUMES"}, - FilePath: "/vela/runtime/volumes", - Name: "runtime.volumes", - Usage: "list of host volumes to mount for the runtime", + Name: "runtime.volumes", + Usage: "list of host volumes to mount for the runtime", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_RUNTIME_VOLUMES"), + cli.EnvVar("RUNTIME_VOLUMES"), + cli.File("/vela/runtime/volumes"), + ), }, &cli.StringSliceFlag{ - EnvVars: []string{"VELA_RUNTIME_DROP_CAPABILITIES", "RUNTIME_DROP_CAPABILITIES"}, - FilePath: "/vela/runtime/drop_capabilities", - Name: "runtime.drop-capabilities", - Usage: "list of kernel capabilities to drop from container privileges (only used by Docker)", + Name: "runtime.drop-capabilities", + Usage: "list of kernel capabilities to drop from container privileges (only used by Docker)", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_RUNTIME_DROP_CAPABILITIES"), + cli.EnvVar("RUNTIME_DROP_CAPABILITIES"), + cli.File("/vela/runtime/drop_capabilities"), + ), }, } From 86a2d9bc513e3f411da1d7bfbd3b01dd63661728 Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Tue, 8 Apr 2025 15:15:42 -0500 Subject: [PATCH 06/63] more v3 --- cmd/vela-worker/run.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/vela-worker/run.go b/cmd/vela-worker/run.go index dbe5e287..3291f54b 100644 --- a/cmd/vela-worker/run.go +++ b/cmd/vela-worker/run.go @@ -8,7 +8,7 @@ import ( "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" _ "github.com/joho/godotenv/autoload" @@ -22,7 +22,7 @@ import ( // run executes the worker based // off the configuration provided. -func run(c *cli.Context) error { +func run(c *cli.Command) error { // set log format for the worker switch c.String("log.format") { case "t", "text", "Text", "TEXT": @@ -119,7 +119,7 @@ func run(c *cli.Context) error { ConfigFile: c.String("runtime.config"), Namespace: c.String("runtime.namespace"), PodsTemplateName: c.String("runtime.pods-template-name"), - PodsTemplateFile: c.Path("runtime.pods-template-file"), + PodsTemplateFile: c.String("runtime.pods-template-file"), HostVolumes: c.StringSlice("runtime.volumes"), PrivilegedImages: c.StringSlice("runtime.privileged-images"), DropCapabilities: c.StringSlice("runtime.drop-capabilities"), From f022c7a5d6c4c65dfb2f497e52901ecedcbd26e1 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Tue, 8 Apr 2025 15:36:31 -0500 Subject: [PATCH 07/63] more changes to v3 --- cmd/vela-worker/run.go | 7 +-- executor/linux/build_test.go | 75 ++++++++++++++++++++++--------- executor/linux/outputs_test.go | 15 ++++--- executor/linux/secret_test.go | 15 ++++--- executor/linux/stage_test.go | 15 ++++--- executor/local/build_test.go | 81 +++++++++++++++++++++++----------- executor/local/outputs_test.go | 16 ++++--- executor/local/stage_test.go | 16 ++++--- 8 files changed, 164 insertions(+), 76 deletions(-) diff --git a/cmd/vela-worker/run.go b/cmd/vela-worker/run.go index 3291f54b..fa0fdc17 100644 --- a/cmd/vela-worker/run.go +++ b/cmd/vela-worker/run.go @@ -3,6 +3,7 @@ package main import ( + "context" "fmt" "net/url" @@ -22,7 +23,7 @@ import ( // run executes the worker based // off the configuration provided. -func run(c *cli.Command) error { +func run(ctx context.Context, c *cli.Command) error { // set log format for the worker switch c.String("log.format") { case "t", "text", "Text", "TEXT": @@ -95,7 +96,7 @@ func run(c *cli.Command) error { }, // build configuration Build: &Build{ - Limit: c.Int("build.limit"), + Limit: int(c.Int("build.limit")), Timeout: c.Duration("build.timeout"), }, // build configuration @@ -103,7 +104,7 @@ func run(c *cli.Command) error { // executor configuration Executor: &executor.Setup{ Driver: c.String("executor.driver"), - MaxLogSize: c.Uint("executor.max_log_size"), + MaxLogSize: uint(c.Uint("executor.max_log_size")), LogStreamingTimeout: c.Duration("executor.log_streaming_timeout"), EnforceTrustedRepos: c.Bool("executor.enforce-trusted-repos"), OutputCtn: outputsCtn, diff --git a/executor/linux/build_test.go b/executor/linux/build_test.go index bf8bcf86..56446428 100644 --- a/executor/linux/build_test.go +++ b/executor/linux/build_test.go @@ -4,7 +4,6 @@ package linux import ( "context" - "flag" "net/http/httptest" "strings" "testing" @@ -13,7 +12,7 @@ import ( "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" logrusTest "github.com/sirupsen/logrus/hooks/test" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" v1 "k8s.io/api/core/v1" "github.com/go-vela/sdk-go/vela" @@ -30,9 +29,15 @@ import ( func TestLinux_CreateBuild(t *testing.T) { // setup types - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - compiler, _ := native.FromCLIContext(cli.NewContext(nil, set, nil)) + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) _build := testBuild() @@ -218,9 +223,15 @@ func TestLinux_CreateBuild(t *testing.T) { func TestLinux_PlanBuild(t *testing.T) { // setup types - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - compiler, _ := native.FromCLIContext(cli.NewContext(nil, set, nil)) + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) _build := testBuild() @@ -400,9 +411,15 @@ func TestLinux_PlanBuild(t *testing.T) { func TestLinux_AssembleBuild(t *testing.T) { // setup types - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - compiler, _ := native.FromCLIContext(cli.NewContext(nil, set, nil)) + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) _build := testBuild() @@ -708,9 +725,15 @@ func TestLinux_AssembleBuild(t *testing.T) { func TestLinux_ExecBuild(t *testing.T) { // setup types - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - compiler, _ := native.FromCLIContext(cli.NewContext(nil, set, nil)) + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) _build := testBuild() @@ -998,9 +1021,15 @@ func TestLinux_ExecBuild(t *testing.T) { func TestLinux_StreamBuild(t *testing.T) { // setup types - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - compiler, _ := native.FromCLIContext(cli.NewContext(nil, set, nil)) + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) _build := testBuild() @@ -1611,9 +1640,15 @@ func TestLinux_StreamBuild(t *testing.T) { func TestLinux_DestroyBuild(t *testing.T) { // setup types - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - compiler, _ := native.FromCLIContext(cli.NewContext(nil, set, nil)) + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) _build := testBuild() diff --git a/executor/linux/outputs_test.go b/executor/linux/outputs_test.go index d832be5c..d8965d6d 100644 --- a/executor/linux/outputs_test.go +++ b/executor/linux/outputs_test.go @@ -4,13 +4,12 @@ package linux import ( "context" - "flag" "net/http/httptest" "os" "testing" "github.com/gin-gonic/gin" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" @@ -277,9 +276,15 @@ func TestLinux_Outputs_delete(t *testing.T) { func TestLinux_Outputs_exec(t *testing.T) { // setup types - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - compiler, _ := native.FromCLIContext(cli.NewContext(nil, set, nil)) + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) _build := testBuild() diff --git a/executor/linux/secret_test.go b/executor/linux/secret_test.go index 79125175..ba0008b0 100644 --- a/executor/linux/secret_test.go +++ b/executor/linux/secret_test.go @@ -4,14 +4,13 @@ package linux import ( "context" - "flag" "net/http/httptest" "os" "testing" "github.com/gin-gonic/gin" "github.com/google/go-cmp/cmp" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" @@ -359,9 +358,15 @@ func TestLinux_Secret_delete(t *testing.T) { func TestLinux_Secret_exec(t *testing.T) { // setup types - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - compiler, _ := native.FromCLIContext(cli.NewContext(nil, set, nil)) + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) _build := testBuild() diff --git a/executor/linux/stage_test.go b/executor/linux/stage_test.go index e6a09134..043f6cda 100644 --- a/executor/linux/stage_test.go +++ b/executor/linux/stage_test.go @@ -5,13 +5,12 @@ package linux import ( "context" "errors" - "flag" "net/http/httptest" "sync" "testing" "github.com/gin-gonic/gin" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" "github.com/go-vela/sdk-go/vela" "github.com/go-vela/server/compiler/native" @@ -28,9 +27,15 @@ func TestLinux_CreateStage(t *testing.T) { _file := "testdata/build/stages/basic.yml" _build := testBuild() - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - compiler, _ := native.FromCLIContext(cli.NewContext(nil, set, nil)) + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) _pipeline, _, err := compiler. Duplicate(). diff --git a/executor/local/build_test.go b/executor/local/build_test.go index ee72c035..71b98039 100644 --- a/executor/local/build_test.go +++ b/executor/local/build_test.go @@ -4,11 +4,10 @@ package local import ( "context" - "flag" "testing" "time" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" "github.com/go-vela/server/compiler/native" "github.com/go-vela/server/compiler/types/pipeline" @@ -18,10 +17,15 @@ import ( func TestLocal_CreateBuild(t *testing.T) { // setup types - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - - compiler, err := native.FromCLIContext(cli.NewContext(nil, set, nil)) + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) if err != nil { t.Errorf("unable to create compiler engine: %v", err) } @@ -98,10 +102,15 @@ func TestLocal_CreateBuild(t *testing.T) { func TestLocal_PlanBuild(t *testing.T) { // setup types - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - - compiler, err := native.FromCLIContext(cli.NewContext(nil, set, nil)) + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) if err != nil { t.Errorf("unable to create compiler engine: %v", err) } @@ -184,10 +193,15 @@ func TestLocal_PlanBuild(t *testing.T) { func TestLocal_AssembleBuild(t *testing.T) { // setup types - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - - compiler, err := native.FromCLIContext(cli.NewContext(nil, set, nil)) + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) if err != nil { t.Errorf("unable to create compiler engine: %v", err) } @@ -304,10 +318,15 @@ func TestLocal_AssembleBuild(t *testing.T) { func TestLocal_ExecBuild(t *testing.T) { // setup types - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - - compiler, err := native.FromCLIContext(cli.NewContext(nil, set, nil)) + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) if err != nil { t.Errorf("unable to create compiler engine: %v", err) } @@ -409,10 +428,15 @@ func TestLocal_ExecBuild(t *testing.T) { func TestLocal_StreamBuild(t *testing.T) { // setup types - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - - compiler, err := native.FromCLIContext(cli.NewContext(nil, set, nil)) + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) if err != nil { t.Errorf("unable to create compiler engine: %v", err) } @@ -626,10 +650,15 @@ func TestLocal_StreamBuild(t *testing.T) { func TestLocal_DestroyBuild(t *testing.T) { // setup types - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - - compiler, err := native.FromCLIContext(cli.NewContext(nil, set, nil)) + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) if err != nil { t.Errorf("unable to create compiler engine: %v", err) } diff --git a/executor/local/outputs_test.go b/executor/local/outputs_test.go index d0b03068..a794adbc 100644 --- a/executor/local/outputs_test.go +++ b/executor/local/outputs_test.go @@ -4,13 +4,12 @@ package local import ( "context" - "flag" "net/http/httptest" "os" "testing" "github.com/gin-gonic/gin" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" @@ -277,10 +276,15 @@ func TestLinux_Outputs_delete(t *testing.T) { func TestLinux_Outputs_exec(t *testing.T) { // setup types - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - compiler, _ := native.FromCLIContext(cli.NewContext(nil, set, nil)) - + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) _build := testBuild() gin.SetMode(gin.TestMode) diff --git a/executor/local/stage_test.go b/executor/local/stage_test.go index 3a759c77..dadae0c1 100644 --- a/executor/local/stage_test.go +++ b/executor/local/stage_test.go @@ -5,11 +5,10 @@ package local import ( "context" "errors" - "flag" "sync" "testing" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" "github.com/go-vela/server/compiler/native" "github.com/go-vela/server/compiler/types/pipeline" @@ -22,10 +21,15 @@ func TestLocal_CreateStage(t *testing.T) { _file := "testdata/build/stages/basic.yml" _build := testBuild() - set := flag.NewFlagSet("test", 0) - set.String("clone-image", "target/vela-git:latest", "doc") - - compiler, err := native.FromCLIContext(cli.NewContext(nil, set, nil)) + cmd := new(cli.Command) + cmd.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "clone-image", + Value: "target/vela-git:latest", + Usage: "doc", + }, + } + compiler, err := native.FromCLICommand(context.Background(), cmd) if err != nil { t.Errorf("unable to create compiler from CLI context: %v", err) } From 060aceb59ca6583376c4c3652993fbe404a8c2eb Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Tue, 8 Apr 2025 16:09:54 -0500 Subject: [PATCH 08/63] chore(deps): update server to current main --- go.mod | 13 +------------ go.sum | 25 +++---------------------- 2 files changed, 4 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index 5b5d2a07..fd611b27 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,6 @@ module github.com/go-vela/worker go 1.24.2 - require ( github.com/Masterminds/semver/v3 v3.3.1 github.com/distribution/reference v0.6.0 @@ -10,14 +9,13 @@ require ( github.com/docker/go-units v0.5.0 github.com/gin-gonic/gin v1.10.0 github.com/go-vela/sdk-go v0.26.2 - github.com/go-vela/server v0.26.4 + github.com/go-vela/server v0.26.5-0.20250408151749-e7e79ba4a47b github.com/golang-jwt/jwt/v5 v5.2.2 github.com/google/go-cmp v0.7.0 github.com/joho/godotenv v1.5.1 github.com/opencontainers/image-spec v1.1.1 github.com/prometheus/client_golang v1.21.1 github.com/sirupsen/logrus v1.9.3 - github.com/urfave/cli/v2 v2.27.6 github.com/urfave/cli/v3 v3.1.1 golang.org/x/sync v0.12.0 gotest.tools/v3 v3.5.2 @@ -34,9 +32,7 @@ require ( github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/go-ini/ini v1.67.0 // indirect github.com/google/go-github/v70 v70.0.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 // indirect @@ -53,10 +49,6 @@ require ( github.com/lib/pq v1.10.9 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect - github.com/minio/crc64nvme v1.0.1 // indirect - github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.89 // indirect - github.com/rs/xid v1.6.0 // indirect github.com/uptrace/opentelemetry-go-extra/otelgorm v0.3.2 // indirect github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect @@ -93,7 +85,6 @@ require ( github.com/containerd/containerd v1.7.11 // indirect github.com/containerd/log v0.1.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -150,7 +141,6 @@ require ( github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/redis/go-redis/v9 v9.7.3 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/afero v1.14.0 // indirect @@ -158,7 +148,6 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect - github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect go.opentelemetry.io/otel v1.35.0 // indirect diff --git a/go.sum b/go.sum index 6f7abf81..2d684e5b 100644 --- a/go.sum +++ b/go.sum @@ -56,8 +56,6 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= -github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -78,8 +76,6 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= @@ -98,8 +94,6 @@ github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= -github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= -github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -123,8 +117,10 @@ github.com/go-playground/validator/v10 v10.25.0 h1:5Dh7cjvzR7BRZadnsVOzPhWsrwUr0 github.com/go-playground/validator/v10 v10.25.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-vela/sdk-go v0.26.2 h1:uqYZNO9VYfDiyXl7MMgUt6XwBwvIj3OWWAIa8q9RuXE= github.com/go-vela/sdk-go v0.26.2/go.mod h1:XLnki5qdk8R+y+J65AOuiJ1pQyBmy7P46E85SPOv0Ww= -github.com/go-vela/server v0.26.4/go.mod h1:SNQVpgPoCmd7dvSA+nILmsFF0Ijoh0nEm24ytJd9J4o= +github.com/go-vela/server v0.26.5-0.20250408151749-e7e79ba4a47b h1:/cK/FLG+CnrUGIqoCSFAqZ+yRQJde6yXPfwQB+PgrS0= +github.com/go-vela/server v0.26.5-0.20250408151749-e7e79ba4a47b/go.mod h1:tsqFasTmpHyIK6WkhL/qNSmxQj/G2ejtoXMNaOSrdys= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -198,7 +194,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= @@ -236,12 +231,6 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= -github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY= -github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.89 h1:hx4xV5wwTUfyv8LarhJAwNecnXpoTsj9v3f3q/ZkiJU= -github.com/minio/minio-go/v7 v7.0.89/go.mod h1:2rFnGAp02p7Dddo1Fq4S2wYOfpF0MUTSeLTRC90I204= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -287,10 +276,6 @@ github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0 github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= -github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= -github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= @@ -325,16 +310,12 @@ github.com/uptrace/opentelemetry-go-extra/otelgorm v0.3.2 h1:Jjn3zoRz13f8b1bR6Lr github.com/uptrace/opentelemetry-go-extra/otelgorm v0.3.2/go.mod h1:wocb5pNrj/sjhWB9J5jctnC0K2eisSdz/nJJBNFHo+A= github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 h1:ZjUj9BLYf9PEqBn8W/OapxhPjVRdC6CsXTdULHsyk5c= github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2/go.mod h1:O8bHQfyinKwTXKkiKNGmLQS7vRsqRxIQTFZpYpHK3IQ= -github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g= -github.com/urfave/cli/v2 v2.27.6/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= github.com/urfave/cli/v3 v3.1.1 h1:bNnl8pFI5dxPOjeONvFCDFoECLQsceDG4ejahs4Jtxk= github.com/urfave/cli/v3 v3.1.1/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZtpYpo= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= -github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From a52952dd72ac0a7a7557cf13c0f02ac381fe7159 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Tue, 8 Apr 2025 21:43:22 -0500 Subject: [PATCH 09/63] point to latest sdk-go --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index fd611b27..cab77d6c 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/docker/docker v27.5.1+incompatible github.com/docker/go-units v0.5.0 github.com/gin-gonic/gin v1.10.0 - github.com/go-vela/sdk-go v0.26.2 + github.com/go-vela/sdk-go v0.26.3-0.20250409022751-0f27f9202249 github.com/go-vela/server v0.26.5-0.20250408151749-e7e79ba4a47b github.com/golang-jwt/jwt/v5 v5.2.2 github.com/google/go-cmp v0.7.0 diff --git a/go.sum b/go.sum index 2d684e5b..4ddc0b53 100644 --- a/go.sum +++ b/go.sum @@ -119,6 +119,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-vela/sdk-go v0.26.2 h1:uqYZNO9VYfDiyXl7MMgUt6XwBwvIj3OWWAIa8q9RuXE= github.com/go-vela/sdk-go v0.26.2/go.mod h1:XLnki5qdk8R+y+J65AOuiJ1pQyBmy7P46E85SPOv0Ww= +github.com/go-vela/sdk-go v0.26.3-0.20250409022751-0f27f9202249 h1:FBRZ2xhYIP1vFT8GdKT2UBtg0T0XURMrfMs903o17fA= +github.com/go-vela/sdk-go v0.26.3-0.20250409022751-0f27f9202249/go.mod h1:LdKoSUviaNe0RzQtgAL/ZAG/hwI2bW0V1FeDY74UUBI= github.com/go-vela/server v0.26.5-0.20250408151749-e7e79ba4a47b h1:/cK/FLG+CnrUGIqoCSFAqZ+yRQJde6yXPfwQB+PgrS0= github.com/go-vela/server v0.26.5-0.20250408151749-e7e79ba4a47b/go.mod h1:tsqFasTmpHyIK6WkhL/qNSmxQj/G2ejtoXMNaOSrdys= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= From ec9dafa183cab4d49489f449853db1af4b63cdac Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Wed, 9 Apr 2025 14:00:17 -0500 Subject: [PATCH 10/63] add back missing code --- cmd/vela-worker/flags.go | 4 ++++ cmd/vela-worker/operate.go | 24 ++++++++++++++++++++++++ cmd/vela-worker/run.go | 15 +++++++++++++++ cmd/vela-worker/worker.go | 3 +++ executor/linux/build.go | 2 +- 5 files changed, 47 insertions(+), 1 deletion(-) diff --git a/cmd/vela-worker/flags.go b/cmd/vela-worker/flags.go index 8f94d8f7..0dcc3cb6 100644 --- a/cmd/vela-worker/flags.go +++ b/cmd/vela-worker/flags.go @@ -5,6 +5,7 @@ package main import ( "context" "fmt" + "github.com/go-vela/server/storage" "strings" "time" @@ -120,5 +121,8 @@ func flags() []cli.Flag { f = append(f, runtime.Flags...) + // Storage Flags + f = append(f, storage.Flags...) + return f } diff --git a/cmd/vela-worker/operate.go b/cmd/vela-worker/operate.go index f517b523..e1b85091 100644 --- a/cmd/vela-worker/operate.go +++ b/cmd/vela-worker/operate.go @@ -76,7 +76,31 @@ func (w *Worker) operate(ctx context.Context) error { // set to error as queue setup fails w.updateWorkerStatus(registryWorker, constants.WorkerStatusError) } + // getting storage creds + logrus.Trace("getting storage s3 creds..") + // fetching queue credentials using registration token + stCreds, _, err := w.VelaClient.Storage.GetInfo() + if err != nil { + logrus.Trace("error getting storage creds") + return err + } + + // if an address was given at start up, use that — else use what is returned from server + if len(w.Config.Executor.Storage.Endpoint) == 0 { + w.Config.Executor.Storage.Endpoint = stCreds.GetStorageAddress() + logrus.Trace("storage address: ", w.Config.Executor.Storage.Driver) + } + + // set access key in storage config + w.Config.Executor.Storage.AccessKey = stCreds.GetAccessKey() + logrus.Trace("access key: ", w.Config.Executor.Storage.AccessKey) + // set secret key in storage config + w.Config.Executor.Storage.SecretKey = stCreds.GetSecretKey() + logrus.Trace("secret key: ", w.Config.Executor.Storage.SecretKey) + // set bucket name in storage config + w.Config.Executor.Storage.Bucket = stCreds.GetStorageBucket() + logrus.Trace("bucket name: ", w.Config.Executor.Storage.Bucket) // spawn goroutine for phoning home executors.Go(func() error { // five second ticker for signal handling diff --git a/cmd/vela-worker/run.go b/cmd/vela-worker/run.go index fa0fdc17..19fde9dd 100644 --- a/cmd/vela-worker/run.go +++ b/cmd/vela-worker/run.go @@ -5,6 +5,7 @@ package main import ( "context" "fmt" + "github.com/go-vela/server/storage" "net/url" "github.com/gin-gonic/gin" @@ -108,6 +109,13 @@ func run(ctx context.Context, c *cli.Command) error { LogStreamingTimeout: c.Duration("executor.log_streaming_timeout"), EnforceTrustedRepos: c.Bool("executor.enforce-trusted-repos"), OutputCtn: outputsCtn, + Storage: &storage.Setup{ + Driver: c.String("storage.driver"), + Endpoint: c.String("storage.endpoint.name"), + AccessKey: c.String("storage.access.key"), + SecretKey: c.String("storage.secret.key"), + Bucket: c.String("storage.bucket.name"), + }, }, // logger configuration Logger: &Logger{ @@ -138,6 +146,13 @@ func run(ctx context.Context, c *cli.Command) error { Address: c.String("server.addr"), Secret: c.String("server.secret"), }, + Storage: &storage.Setup{ + Driver: c.String("storage.driver"), + Endpoint: c.String("storage.endpoint.name"), + AccessKey: c.String("storage.access.key"), + SecretKey: c.String("storage.secret.key"), + Bucket: c.String("storage.bucket.name"), + }, // Certificate configuration Certificate: &Certificate{ Cert: c.String("server.cert"), diff --git a/cmd/vela-worker/worker.go b/cmd/vela-worker/worker.go index 834a88b0..8c4298c0 100644 --- a/cmd/vela-worker/worker.go +++ b/cmd/vela-worker/worker.go @@ -3,6 +3,7 @@ package main import ( + "github.com/go-vela/server/storage" "net/url" "sync" "time" @@ -57,6 +58,7 @@ type ( Server *Server Certificate *Certificate TLSMinVersion string + Storage *storage.Setup } // Worker represents all configuration and @@ -72,5 +74,6 @@ type ( RunningBuilds []*api.Build QueueCheckedIn bool RunningBuildsMutex sync.Mutex + Storage storage.Storage } ) diff --git a/executor/linux/build.go b/executor/linux/build.go index a5f20a89..13719c19 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -547,7 +547,7 @@ func (c *client) ExecBuild(ctx context.Context) error { } _step.Secrets = append(_step.Secrets, sec) } - + // logic for polling files only if the test-report step is present // iterate through the steps in the build for _, s := range c.pipeline.Steps { From f42e676b7aa58d3b0242a22d2e9be19f01c7b2a9 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Wed, 9 Apr 2025 17:42:11 -0500 Subject: [PATCH 11/63] add back missing code --- cmd/vela-worker/exec.go | 4 ++++ cmd/vela-worker/operate.go | 2 +- executor/linux/build.go | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index 32bd7ca6..0c2b3499 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -150,6 +150,9 @@ func (w *Worker) exec(index int, config *api.Worker) error { execOutputCtn := *w.Config.Executor.OutputCtn execOutputCtn.ID = fmt.Sprintf("outputs_%s", p.ID) + // dereference configured storage config and set the storage config for the executor + execStorage := *w.Config.Executor.Storage + // create logger with extra metadata // // https://pkg.go.dev/github.com/sirupsen/logrus#WithFields @@ -242,6 +245,7 @@ func (w *Worker) exec(index int, config *api.Worker) error { Pipeline: p.Sanitize(w.Config.Runtime.Driver), Version: v.Semantic(), OutputCtn: &execOutputCtn, + Storage: &execStorage, }) // add the executor to the worker diff --git a/cmd/vela-worker/operate.go b/cmd/vela-worker/operate.go index e1b85091..506262cf 100644 --- a/cmd/vela-worker/operate.go +++ b/cmd/vela-worker/operate.go @@ -70,7 +70,7 @@ func (w *Worker) operate(ctx context.Context) error { // setup the queue // // https://pkg.go.dev/github.com/go-vela/server/queue#New - w.Queue, err = queue.New(w.Config.Queue) + w.Queue, err = queue.New(gctx, w.Config.Queue) if err != nil { logrus.Error("queue setup failed") // set to error as queue setup fails diff --git a/executor/linux/build.go b/executor/linux/build.go index 13719c19..a5f20a89 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -547,7 +547,7 @@ func (c *client) ExecBuild(ctx context.Context) error { } _step.Secrets = append(_step.Secrets, sec) } - + // logic for polling files only if the test-report step is present // iterate through the steps in the build for _, s := range c.pipeline.Steps { From 76529b1169c606a49774b2cb641593d62f8cda03 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Fri, 11 Apr 2025 13:34:04 -0500 Subject: [PATCH 12/63] upload with build number --- executor/linux/build.go | 4 ++-- executor/linux/outputs.go | 35 +++++++++++++++++++++++------------ 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/executor/linux/build.go b/executor/linux/build.go index a5f20a89..9fa278df 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -554,11 +554,11 @@ func (c *client) ExecBuild(ctx context.Context) error { c.Logger.Infof("polling files for %s step", s.Name) if len(s.TestReport.Results) != 0 { - err := c.outputs.pollFiles(ctx, c.OutputCtn, s.TestReport.Results) + err := c.outputs.pollFiles(ctx, c.OutputCtn, s.TestReport.Results, c.build) c.Logger.Errorf("unable to poll files for results: %v", err) } if len(s.TestReport.Attachments) != 0 { - err := c.outputs.pollFiles(ctx, c.OutputCtn, s.TestReport.Attachments) + err := c.outputs.pollFiles(ctx, c.OutputCtn, s.TestReport.Attachments, c.build) c.Logger.Errorf("unable to poll files for attachments: %v", err) } diff --git a/executor/linux/outputs.go b/executor/linux/outputs.go index 564b8015..ee19eae3 100644 --- a/executor/linux/outputs.go +++ b/executor/linux/outputs.go @@ -8,11 +8,11 @@ import ( "encoding/base64" "fmt" api "github.com/go-vela/server/api/types" - + "github.com/go-vela/server/compiler/types/pipeline" envparse "github.com/hashicorp/go-envparse" "github.com/sirupsen/logrus" - - "github.com/go-vela/server/compiler/types/pipeline" + "path/filepath" + "strconv" ) // outputSvc handles communication with the outputs container during the build. @@ -151,7 +151,7 @@ func (o *outputSvc) poll(ctx context.Context, ctn *pipeline.Container) (map[stri } // pollFiles tails the output for sidecar container. -func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, fileList []string) error { +func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, fileList []string, b *api.Build) error { // exit if outputs container has not been configured if len(ctn.Image) == 0 { return nil @@ -165,31 +165,42 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, file logger.Debug("tailing container") logger.Debugf("fileList: %v", fileList) // grab outputs - fileNames, err := o.client.Runtime.PollFileNames(ctx, ctn, fileList) + filesPath, err := o.client.Runtime.PollFileNames(ctx, ctn, fileList) if err != nil { return fmt.Errorf("unable to poll file names: %v", err) } - if len(fileNames) != 0 { - for _, fileName := range fileNames { + if len(filesPath) != 0 { + for _, filePath := range filesPath { + fileName := filepath.Base(filePath) logger.Infof("fileName: %v", fileName) - reader, size, err := o.client.Runtime.PollFileContent(ctx, ctn, fileName) + logger.Infof("filePath: %v", filePath) + reader, size, err := o.client.Runtime.PollFileContent(ctx, ctn, filePath) if err != nil { logger.Errorf("unable to poll file content: %v", err) return err } - + // err = o.client.Storage.UploadObject(ctx, &api.Object{ - ObjectName: fileName, + ObjectName: fmt.Sprintf(b.GetRepo().GetOrg()+"/"+b.GetRepo().GetName()+"/"+strconv.FormatInt(b.GetID(), 10)+"/%s", fileName), Bucket: api.Bucket{BucketName: o.client.Storage.GetBucket(ctx)}, - FilePath: fileName, + FilePath: filePath, }, reader, size) if err != nil { logger.Errorf("unable to upload object: %v", err) return err } + //err = o.client.Storage.Upload(ctx, &api.Object{ + // ObjectName: fmt.Sprintf(b.GetRepo().GetOrg()+"/"+b.GetRepo().GetName()+"/"+strconv.FormatInt(b.GetID(), 10)+"/%s", fileName), + // Bucket: api.Bucket{BucketName: o.client.Storage.GetBucket(ctx)}, + // FilePath: filePath, + //}) + //if err != nil { + // logger.Errorf("unable to upload object: %v", err) + // return err + //} } - logger.Infof("fileNames: %v", fileNames) + return nil } logger.Debug("no files found") From c26d8508da9ea743d04707067381d9047ee2e344 Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Tue, 17 Jun 2025 15:18:33 -0500 Subject: [PATCH 13/63] feat(test reports): try testreport add db record --- executor/linux/build.go | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/executor/linux/build.go b/executor/linux/build.go index 2c578430..2985becf 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -553,13 +553,26 @@ func (c *client) ExecBuild(ctx context.Context) error { for _, s := range c.pipeline.Steps { c.Logger.Infof("polling files for %s step", s.Name) - if len(s.TestReport.Results) != 0 { - err := c.outputs.pollFiles(ctx, c.OutputCtn, s.TestReport.Results, c.build) - c.Logger.Errorf("unable to poll files for results: %v", err) - } - if len(s.TestReport.Attachments) != 0 { - err := c.outputs.pollFiles(ctx, c.OutputCtn, s.TestReport.Attachments, c.build) - c.Logger.Errorf("unable to poll files for attachments: %v", err) + if !s.TestReport.Empty() { + c.Logger.Debug("creating test report record in database") + // send API call to update the test report + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#TestReportService.Add + // TODO: .Add should be .Update + // TODO: handle somewhere if multiple test report keys exist in pipeline + tr, resp, err := c.Vela.TestReport.Add(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber()) + if err != nil { + c.Logger.Errorf("unable to create test report record in databases: %v, %v, %v", tr.GetBuildID(), resp.StatusCode, err) + } + + if len(s.TestReport.Results) != 0 { + err := c.outputs.pollFiles(ctx, c.OutputCtn, s.TestReport.Results, c.build) + c.Logger.Errorf("unable to poll files for results: %v", err) + } + if len(s.TestReport.Attachments) != 0 { + err := c.outputs.pollFiles(ctx, c.OutputCtn, s.TestReport.Attachments, c.build) + c.Logger.Errorf("unable to poll files for attachments: %v", err) + } } } From 9e63c7b014c8c553f6dbe97a384e4611722c1ead Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Thu, 3 Jul 2025 10:54:47 -0500 Subject: [PATCH 14/63] update docker package --- go.mod | 9 ++++++++- go.sum | 19 ++++++++++--------- runtime/docker/test_report.go | 10 +++++----- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 7063cafd..0511cd20 100644 --- a/go.mod +++ b/go.mod @@ -35,9 +35,11 @@ require ( github.com/containerd/containerd/v2 v2.1.3 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/expr-lang/expr v1.17.5 // indirect github.com/fxamacker/cbor/v2 v2.8.0 // indirect - github.com/google/go-github/v73 v73.0.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/google/go-github/v72 v72.0.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect github.com/invopop/jsonschema v0.13.0 // indirect @@ -47,13 +49,18 @@ require ( github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/lestrrat-go/httprc/v3 v3.0.0 // indirect github.com/lestrrat-go/jwx/v3 v3.0.8 // indirect github.com/lestrrat-go/option/v2 v2.0.0 // indirect github.com/lib/pq v1.10.9 // indirect github.com/mattn/go-sqlite3 v1.14.28 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect + github.com/minio/crc64nvme v1.0.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.89 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/uptrace/opentelemetry-go-extra/otelgorm v0.3.2 // indirect github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect diff --git a/go.sum b/go.sum index 56cd4da5..7a8162b1 100644 --- a/go.sum +++ b/go.sum @@ -100,6 +100,8 @@ github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -123,10 +125,6 @@ github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/go-vela/sdk-go v0.27.0-rc1 h1:h6awgY5rxeauogHEWJdWJe3XLtHpnt8xe/dbXwtpivU= -github.com/go-vela/sdk-go v0.27.0-rc1/go.mod h1:213YVYZcdAhi06c+DOXe4wxEtvstM1mSOHjK+DFk0vg= -github.com/go-vela/server v0.27.0-rc1 h1:dUYjeSEzFuzl4Vg2B+lA6Ub1BxLjxjnQi4BzqQ2TMB8= -github.com/go-vela/server v0.27.0-rc1/go.mod h1:/CqDivw0lC3fyNKZSgxdCGh+4WkZr2MbjzyT7FWyOfE= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -144,8 +142,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-github/v73 v73.0.0 h1:aR+Utnh+Y4mMkS+2qLQwcQ/cF9mOTpdwnzlaw//rG24= -github.com/google/go-github/v73 v73.0.0/go.mod h1:fa6w8+/V+edSU0muqdhCVY7Beh1M8F1IlQPZIANKIYw= +github.com/google/go-github/v72 v72.0.0 h1:FcIO37BLoVPBO9igQQ6tStsv2asG4IPcYFi655PPvBM= +github.com/google/go-github/v72 v72.0.0/go.mod h1:WWtw8GMRiL62mvIquf1kO3onRHeWWKmK01qdCY8c5fg= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -239,10 +237,12 @@ github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEu github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY= +github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.83 h1:W4Kokksvlz3OKf3OqIlzDNKd4MERlC2oN8YptwJ0+GA= -github.com/minio/minio-go/v7 v7.0.83/go.mod h1:57YXpvc5l3rjPdhqNrDsvVlY0qPI6UTk1bflAe+9doY= +github.com/minio/minio-go/v7 v7.0.89 h1:hx4xV5wwTUfyv8LarhJAwNecnXpoTsj9v3f3q/ZkiJU= +github.com/minio/minio-go/v7 v7.0.89/go.mod h1:2rFnGAp02p7Dddo1Fq4S2wYOfpF0MUTSeLTRC90I204= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -277,8 +277,9 @@ github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= diff --git a/runtime/docker/test_report.go b/runtime/docker/test_report.go index 5c5fa55d..e23aca5d 100644 --- a/runtime/docker/test_report.go +++ b/runtime/docker/test_report.go @@ -5,7 +5,7 @@ import ( "context" "encoding/base64" "fmt" - "github.com/docker/docker/api/types" + dockerContainerTypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/pkg/stdcopy" "github.com/go-vela/server/compiler/types/pipeline" "io" @@ -28,7 +28,7 @@ func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, pat dir, filename := filepath.Split(path) c.Logger.Tracef("searching for file %s in %s", filename, dir) - execConfig := types.ExecConfig{ + execConfig := dockerContainerTypes.ExecOptions{ Tty: true, //Cmd: []string{"sh", "-c", fmt.Sprintf("find %s -type f -name %s", dir, filename)}, Cmd: []string{"sh", "-c", fmt.Sprintf("find / -type f -path *%s -print", path)}, @@ -43,7 +43,7 @@ func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, pat return nil, err } - hijackedResponse, err := c.Docker.ContainerExecAttach(ctx, responseExec.ID, types.ExecStartCheck{}) + hijackedResponse, err := c.Docker.ContainerExecAttach(ctx, responseExec.ID, dockerContainerTypes.ExecAttachOptions{}) if err != nil { c.Logger.Errorf("unable to attach to exec for container: %v", err) return nil, err @@ -117,7 +117,7 @@ func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, p return nil, 0, nil } cmd := []string{"sh", "-c", fmt.Sprintf("base64 %s", path)} - execConfig := types.ExecConfig{ + execConfig := dockerContainerTypes.ExecOptions{ Cmd: cmd, AttachStdout: true, AttachStderr: false, @@ -129,7 +129,7 @@ func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, p if err != nil { return nil, 0, fmt.Errorf("failed to create exec instance: %w", err) } - resp, err := c.Docker.ContainerExecAttach(ctx, execID.ID, types.ExecStartCheck{}) + resp, err := c.Docker.ContainerExecAttach(ctx, execID.ID, dockerContainerTypes.ExecAttachOptions{}) if err != nil { return nil, 0, fmt.Errorf("failed to attach to exec instance: %w", err) } From 54eccc641916383e0158ddd2583a9dd166ff9758 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Mon, 14 Jul 2025 13:59:40 -0500 Subject: [PATCH 15/63] refactor code --- executor/linux/build.go | 36 +++---- executor/linux/outputs.go | 91 ++++++----------- executor/local/outputs.go | 2 +- runtime/docker/test_report.go | 179 +++++++++++++++++----------------- 4 files changed, 140 insertions(+), 168 deletions(-) diff --git a/executor/linux/build.go b/executor/linux/build.go index 2985becf..397a8806 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -550,31 +550,31 @@ func (c *client) ExecBuild(ctx context.Context) error { // logic for polling files only if the test-report step is present // iterate through the steps in the build - for _, s := range c.pipeline.Steps { - c.Logger.Infof("polling files for %s step", s.Name) - if !s.TestReport.Empty() { - c.Logger.Debug("creating test report record in database") - // send API call to update the test report - // - // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#TestReportService.Add - // TODO: .Add should be .Update - // TODO: handle somewhere if multiple test report keys exist in pipeline - tr, resp, err := c.Vela.TestReport.Add(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber()) - if err != nil { - c.Logger.Errorf("unable to create test report record in databases: %v, %v, %v", tr.GetBuildID(), resp.StatusCode, err) - } + if !_step.TestReport.Empty() { + c.Logger.Debug("creating test report record in database") + // send API call to update the test report + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#TestReportService.Add + // TODO: .Add should be .Update + // TODO: handle somewhere if multiple test report keys exist in pipeline + tr, resp, err := c.Vela.TestReport.Add(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber()) + if err != nil { + c.Logger.Errorf("unable to create test report record in databases: %v, %v, %v", tr.GetBuildID(), resp.StatusCode, err) + } - if len(s.TestReport.Results) != 0 { - err := c.outputs.pollFiles(ctx, c.OutputCtn, s.TestReport.Results, c.build) + if len(_step.TestReport.Results) != 0 { + err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.TestReport.Results, c.build) + if err != nil { c.Logger.Errorf("unable to poll files for results: %v", err) } - if len(s.TestReport.Attachments) != 0 { - err := c.outputs.pollFiles(ctx, c.OutputCtn, s.TestReport.Attachments, c.build) + } + if len(_step.TestReport.Attachments) != 0 { + err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.TestReport.Attachments, c.build) + if err != nil { c.Logger.Errorf("unable to poll files for attachments: %v", err) } } - } // perform any substitution on dynamic variables diff --git a/executor/linux/outputs.go b/executor/linux/outputs.go index 2c712f3f..9024553a 100644 --- a/executor/linux/outputs.go +++ b/executor/linux/outputs.go @@ -151,11 +151,11 @@ func (o *outputSvc) poll(ctx context.Context, ctn *pipeline.Container) (map[stri return outputMap, maskMap, nil } -// pollFiles tails the output for sidecar container. +// pollFiles polls the output for files from the sidecar container. func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, fileList []string, b *api.Build) error { // exit if outputs container has not been configured if len(ctn.Image) == 0 { - return nil + return fmt.Errorf("no outputs container configured") } // update engine logger with outputs metadata @@ -163,69 +163,38 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, file // https://pkg.go.dev/github.com/sirupsen/logrus#Entry.WithField logger := o.client.Logger.WithField("test-outputs", ctn.Name) - logger.Debug("tailing container") - logger.Debugf("fileList: %v", fileList) - // grab outputs + logger.Debug("polling files from container") + + // grab file paths from the container filesPath, err := o.client.Runtime.PollFileNames(ctx, ctn, fileList) if err != nil { - return fmt.Errorf("unable to poll file names: %v", err) - } - if len(filesPath) != 0 { - for _, filePath := range filesPath { - fileName := filepath.Base(filePath) - logger.Infof("fileName: %v", fileName) - logger.Infof("filePath: %v", filePath) - reader, size, err := o.client.Runtime.PollFileContent(ctx, ctn, filePath) - if err != nil { - logger.Errorf("unable to poll file content: %v", err) - return err - } - // - err = o.client.Storage.UploadObject(ctx, &api.Object{ - ObjectName: fmt.Sprintf(b.GetRepo().GetOrg()+"/"+b.GetRepo().GetName()+"/"+strconv.FormatInt(b.GetNumber(), 10)+"/%s", fileName), - Bucket: api.Bucket{BucketName: o.client.Storage.GetBucket(ctx)}, - FilePath: filePath, - }, reader, size) - if err != nil { - logger.Errorf("unable to upload object: %v", err) - return err - } - - //err = o.client.Storage.Upload(ctx, &api.Object{ - // ObjectName: fmt.Sprintf(b.GetRepo().GetOrg()+"/"+b.GetRepo().GetName()+"/"+strconv.FormatInt(b.GetNumber(), 10)+"/%s", fileName), - // Bucket: api.Bucket{BucketName: o.client.Storage.GetBucket(ctx)}, - // FilePath: filePath, - //}) - //if err != nil { - // logger.Errorf("unable to upload object: %v", err) - // return err - //} - } + return fmt.Errorf("unable to poll file names: %w", err) + } - return nil + if len(filesPath) == 0 { + return fmt.Errorf("no files found for file list: %v", fileList) } - logger.Debug("no files found") - return fmt.Errorf("no files found: %v", err) - //reader := bytes.NewReader(outputBytes) - // - //outputMap, err := envparse.Parse(reader) - //if err != nil { - // logger.Debugf("unable to parse output map: %v", err) - //} - // - //// grab masked outputs - //maskedBytes, err := o.client.Runtime.PollOutputsContainer(ctx, ctn, "/vela/outputs/masked.env") - //if err != nil { - // return nil, nil, err - //} - // - //reader = bytes.NewReader(maskedBytes) - // - //maskMap, err := envparse.Parse(reader) - //if err != nil { - // logger.Debugf("unable to parse masked output map: %v", err) - //} + // process each file found + for _, filePath := range filesPath { + fileName := filepath.Base(filePath) + + // get file content from container + reader, size, err := o.client.Runtime.PollFileContent(ctx, ctn, filePath) + if err != nil { + return fmt.Errorf("unable to poll file content for %s: %w", filePath, err) + } - //return outputMap, maskMap, nil + // upload file to storage + err = o.client.Storage.UploadObject(ctx, &api.Object{ + ObjectName: fmt.Sprintf(b.GetRepo().GetOrg()+"/"+b.GetRepo().GetName()+"/"+strconv.FormatInt(b.GetNumber(), 10)+"/%s", fileName), + Bucket: api.Bucket{BucketName: o.client.Storage.GetBucket(ctx)}, + FilePath: filePath, + }, reader, size) + if err != nil { + return fmt.Errorf("unable to upload object: %v", err) + } + } + + return nil } diff --git a/executor/local/outputs.go b/executor/local/outputs.go index f0922b6f..d55e519c 100644 --- a/executor/local/outputs.go +++ b/executor/local/outputs.go @@ -110,7 +110,7 @@ func (o *outputSvc) poll(ctx context.Context, ctn *pipeline.Container) (map[stri outputMap, err := envparse.Parse(reader) if err != nil { - logrus.Debugf("unable to parse output map: %v", err) + logrus.Debugf("unable to parse local output map: %v", err) } // grab masked outputs diff --git a/runtime/docker/test_report.go b/runtime/docker/test_report.go index e23aca5d..edee8e95 100644 --- a/runtime/docker/test_report.go +++ b/runtime/docker/test_report.go @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 + package docker import ( @@ -5,108 +7,105 @@ import ( "context" "encoding/base64" "fmt" - dockerContainerTypes "github.com/docker/docker/api/types/container" - "github.com/docker/docker/pkg/stdcopy" - "github.com/go-vela/server/compiler/types/pipeline" "io" "path/filepath" "strings" + + dockerContainerTypes "github.com/docker/docker/api/types/container" + "github.com/docker/docker/pkg/stdcopy" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/constants" ) +// isAllowedExt returns true if ext (".xml", ".png", etc.) is in your allow-list. +func isAllowedExt(ext string) bool { + ext = strings.ToLower(ext) + for _, a := range constants.AllAllowedExtensions { + if ext == a { + return true + } + } + return false +} + +// execContainerLines runs `sh -c cmd` in the named container and +// returns its stdout split by newline (error if anything on stderr). +func (c *client) execContainerLines(ctx context.Context, containerID, cmd string) ([]string, error) { + execConfig := dockerContainerTypes.ExecOptions{ + Tty: true, + Cmd: []string{"sh", "-c", cmd}, + AttachStdout: true, + AttachStderr: true, + } + resp, err := c.Docker.ContainerExecCreate(ctx, containerID, execConfig) + if err != nil { + return nil, fmt.Errorf("create exec: %w", err) + } + attach, err := c.Docker.ContainerExecAttach(ctx, resp.ID, dockerContainerTypes.ExecAttachOptions{}) + if err != nil { + return nil, fmt.Errorf("attach exec: %w", err) + } + defer attach.Close() + + var outBuf, errBuf bytes.Buffer + if _, err := stdcopy.StdCopy(&outBuf, &errBuf, attach.Reader); err != nil { + return nil, fmt.Errorf("copy exec output: %w", err) + } + if errBuf.Len() > 0 { + return nil, fmt.Errorf("exec error: %s", errBuf.String()) + } + lines := strings.Split(strings.TrimSpace(outBuf.String()), "\n") + return lines, nil +} + // PollFileNames grabs files name from provided path // within a container and uploads them to s3 func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, paths []string) ([]string, error) { - c.Logger.Tracef("gathering test results and attachments from container %s", ctn.ID) - - var fullFilePaths []string - if len(ctn.Image) == 0 { + c.Logger.Tracef("gathering files from container %s", ctn.ID) + if ctn.Image == "" { return nil, nil } - // iterate through the steps in the build - // iterate through the results paths and store them in the map - for _, path := range paths { - dir, filename := filepath.Split(path) - c.Logger.Tracef("searching for file %s in %s", filename, dir) - - execConfig := dockerContainerTypes.ExecOptions{ - Tty: true, - //Cmd: []string{"sh", "-c", fmt.Sprintf("find %s -type f -name %s", dir, filename)}, - Cmd: []string{"sh", "-c", fmt.Sprintf("find / -type f -path *%s -print", path)}, - AttachStderr: true, - AttachStdout: true, - } - c.Logger.Infof("executing command: %v", execConfig.Cmd) - responseExec, err := c.Docker.ContainerExecCreate(ctx, ctn.ID, execConfig) - if err != nil { - c.Logger.Errorf("unable to create exec for container: %v", err) - return nil, err - } + results := make([]string, 0) + for _, p := range paths { + // use find on the container to locate candidates + cmd := fmt.Sprintf("find / -type f -path '*%s' -print", p) + c.Logger.Infof("running: %s", cmd) - hijackedResponse, err := c.Docker.ContainerExecAttach(ctx, responseExec.ID, dockerContainerTypes.ExecAttachOptions{}) + lines, err := c.execContainerLines(ctx, ctn.ID, cmd) if err != nil { - c.Logger.Errorf("unable to attach to exec for container: %v", err) - return nil, err + return nil, fmt.Errorf("searching %q: %w", p, err) } + c.Logger.Tracef("candidates: %v", lines) - defer func() { - if hijackedResponse.Conn != nil { - hijackedResponse.Close() + for _, raw := range lines { + // 1) strip whitespace (including CR/LF) and normalize the path + fp := filepath.Clean(strings.TrimSpace(raw)) + if fp == "" { + continue } - }() - outputStdout := new(bytes.Buffer) - outputStderr := new(bytes.Buffer) - - if hijackedResponse.Reader != nil { - _, err := stdcopy.StdCopy(outputStdout, outputStderr, hijackedResponse.Reader) - if err != nil { - c.Logger.Errorf("unable to copy logs for container: %v", err) + // 2) quick ext check + ext := strings.ToLower(filepath.Ext(fp)) + if !isAllowedExt(ext) { + c.Logger.Infof("skipping %s (ext %s not allowed)", fp, ext) + continue } - } - if outputStderr.Len() > 0 { - return nil, fmt.Errorf("error: %s", outputStderr.String()) + // 3) accept + c.Logger.Infof("accepted file: %s", fp) + results = append(results, fp) } + } - data := outputStdout.String() - c.Logger.Infof("found files: %s", data) - - filePaths := strings.Split(data, "\n") - for _, filePath := range filePaths { - if filePath != "" { - - fullFilePaths = append(fullFilePaths, strings.TrimSpace(filePath)) - c.Logger.Infof("full file: %s", filePath) - } - } + // Add logging for empty results in PollFileNames + if len(results) == 0 { + c.Logger.Errorf("PollFileNames found no matching files for paths: %v", paths) + return results, fmt.Errorf("no matching files found for any paths %v", paths) } - if len(fullFilePaths) == 0 { - return nil, fmt.Errorf("no matching files found for any provided paths") - } - - return fullFilePaths, nil - - // iterate through the steps in the build - //for _, step := range p.Steps { - // if len(step.TestReport.Results) == 0 { - // c.Logger.Warnf("no results provided for the step %s", step.ID) - // return fmt.Errorf("no results provided for the step %s", step.ID) - // } - // if len(step.TestReport.Attachments) == 0 { - // c.Logger.Warnf("no attachments provided for the step %s", step.ID) - // return fmt.Errorf("no attachments provided for the step %s", step.ID) - // } - // // check if the step has the provided paths from results - // for _, result := range step.TestReport.Results { - // _, err := os.Stat(result) - // if err != nil { - // c.Logger.Errorf("unable to find test result %s for step %s", result, step.ID) - // continue - // } - // } - //} - //return nil + return results, nil + } // PollFileContent retrieves the content and size of a file inside a container. @@ -114,7 +113,8 @@ func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, p c.Logger.Tracef("gathering test results and attachments from container %s", ctn.ID) if len(ctn.Image) == 0 { - return nil, 0, nil + // return an empty reader instead of nil + return bytes.NewReader(nil), 0, fmt.Errorf("empty container image") } cmd := []string{"sh", "-c", fmt.Sprintf("base64 %s", path)} execConfig := dockerContainerTypes.ExecOptions{ @@ -127,10 +127,12 @@ func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, p c.Logger.Infof("executing command for content: %v", execConfig.Cmd) execID, err := c.Docker.ContainerExecCreate(ctx, ctn.ID, execConfig) if err != nil { + c.Logger.Debugf("PollFileContent exec-create failed for %q: %v", path, err) return nil, 0, fmt.Errorf("failed to create exec instance: %w", err) } resp, err := c.Docker.ContainerExecAttach(ctx, execID.ID, dockerContainerTypes.ExecAttachOptions{}) if err != nil { + c.Logger.Debugf("PollFileContent exec-attach failed for %q: %v", path, err) return nil, 0, fmt.Errorf("failed to attach to exec instance: %w", err) } @@ -155,16 +157,17 @@ func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, p } data := outputStdout.Bytes() + // Add logging for empty data in PollFileContent + if len(data) == 0 { + c.Logger.Errorf("PollFileContent returned no data for path: %s", path) + return nil, 0, fmt.Errorf("no data returned from base64 command") + } + decoded, err := base64.StdEncoding.DecodeString(string(data)) if err != nil { c.Logger.Errorf("unable to decode base64 data: %v", err) - return nil, 0, err + return nil, 0, fmt.Errorf("failed to decode base64 data: %w", err) } - //c.Logger.Infof("data: %v", string(data)) - // convert the data to a reader - reader := bytes.NewReader(decoded) - // get the size of the data - size := int64(len(decoded)) - return reader, size, nil + return bytes.NewReader(decoded), int64(len(decoded)), nil } From 02bc917e762966b2a79a64b32770adc112cc2361 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Tue, 15 Jul 2025 15:43:49 -0500 Subject: [PATCH 16/63] add testattachment --- executor/linux/build.go | 21 ++++++++---- executor/linux/outputs.go | 25 +++++++++++--- executor/linux/report.go | 31 +++++++++++++++++ executor/linux/testattachments.go | 56 +++++++++++++++++++++++++++++++ runtime/docker/test_report.go | 46 ++++++++++++------------- 5 files changed, 145 insertions(+), 34 deletions(-) create mode 100644 executor/linux/report.go create mode 100644 executor/linux/testattachments.go diff --git a/executor/linux/build.go b/executor/linux/build.go index 397a8806..66f4332a 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -450,6 +450,12 @@ func (c *client) ExecBuild(ctx context.Context) error { // output maps for dynamic environment variables captured from volume var opEnv, maskEnv map[string]string + // test report object for storing the test report information + var tr *api.TestReport + + // Flag to track if we've already created the test report record + testReportCreated := false + // fire up output container to run with the build c.Logger.Infof("creating outputs container %s", c.OutputCtn.ID) @@ -558,19 +564,22 @@ func (c *client) ExecBuild(ctx context.Context) error { // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#TestReportService.Add // TODO: .Add should be .Update // TODO: handle somewhere if multiple test report keys exist in pipeline - tr, resp, err := c.Vela.TestReport.Add(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber()) - if err != nil { - c.Logger.Errorf("unable to create test report record in databases: %v, %v, %v", tr.GetBuildID(), resp.StatusCode, err) - } + if !testReportCreated { + tr, c.err = c.CreateTestReport() + if c.err != nil { + return fmt.Errorf("unable to create test report: %w", c.err) + } + testReportCreated = true + } if len(_step.TestReport.Results) != 0 { - err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.TestReport.Results, c.build) + err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.TestReport.Results, c.build, tr) if err != nil { c.Logger.Errorf("unable to poll files for results: %v", err) } } if len(_step.TestReport.Attachments) != 0 { - err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.TestReport.Attachments, c.build) + err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.TestReport.Attachments, c.build, tr) if err != nil { c.Logger.Errorf("unable to poll files for attachments: %v", err) } diff --git a/executor/linux/outputs.go b/executor/linux/outputs.go index 9024553a..244470e1 100644 --- a/executor/linux/outputs.go +++ b/executor/linux/outputs.go @@ -152,7 +152,7 @@ func (o *outputSvc) poll(ctx context.Context, ctn *pipeline.Container) (map[stri } // pollFiles polls the output for files from the sidecar container. -func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, fileList []string, b *api.Build) error { +func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, fileList []string, b *api.Build, tr *api.TestReport) error { // exit if outputs container has not been configured if len(ctn.Image) == 0 { return fmt.Errorf("no outputs container configured") @@ -163,8 +163,6 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, file // https://pkg.go.dev/github.com/sirupsen/logrus#Entry.WithField logger := o.client.Logger.WithField("test-outputs", ctn.Name) - logger.Debug("polling files from container") - // grab file paths from the container filesPath, err := o.client.Runtime.PollFileNames(ctx, ctn, fileList) if err != nil { @@ -178,6 +176,7 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, file // process each file found for _, filePath := range filesPath { fileName := filepath.Base(filePath) + logger.Debugf("processing file: %s (path: %s)", fileName, filePath) // get file content from container reader, size, err := o.client.Runtime.PollFileContent(ctx, ctn, filePath) @@ -185,14 +184,30 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, file return fmt.Errorf("unable to poll file content for %s: %w", filePath, err) } + // create storage object path + objectName := fmt.Sprintf("%s/%s/%s/%s", + b.GetRepo().GetOrg(), + b.GetRepo().GetName(), + strconv.FormatInt(b.GetNumber(), 10), + fileName) + // upload file to storage err = o.client.Storage.UploadObject(ctx, &api.Object{ - ObjectName: fmt.Sprintf(b.GetRepo().GetOrg()+"/"+b.GetRepo().GetName()+"/"+strconv.FormatInt(b.GetNumber(), 10)+"/%s", fileName), + ObjectName: objectName, Bucket: api.Bucket{BucketName: o.client.Storage.GetBucket(ctx)}, FilePath: filePath, }, reader, size) if err != nil { - return fmt.Errorf("unable to upload object: %v", err) + return fmt.Errorf("unable to upload object %s: %w", fileName, err) + } + + logger.Debugf("successfully uploaded file %s (%d bytes)", fileName, size) + + // create test attachment record in database after successful upload + err = o.client.CreateTestAttachment(fileName, filePath, size, tr) + if err != nil { + logger.Errorf("unable to create test attachment record for %s: %v", fileName, err) + // don't return error here to avoid blocking the upload process } } diff --git a/executor/linux/report.go b/executor/linux/report.go new file mode 100644 index 00000000..44543abc --- /dev/null +++ b/executor/linux/report.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 + +package linux + +import ( + "fmt" + + api "github.com/go-vela/server/api/types" +) + +// CreateTestReport creates a test report record in the database for the current build. +func (c *client) CreateTestReport() (*api.TestReport, error) { + // create empty test report for the build + testReport := &api.TestReport{} + + // update test report in database + tr, resp, err := c.Vela.TestReport.Update( + c.build.GetRepo().GetOrg(), + c.build.GetRepo().GetName(), + c.build.GetNumber(), + testReport, + ) + if err != nil { + return nil, fmt.Errorf("failed to create test report record: build=%d, status=%d, error=%w", + c.build.GetNumber(), resp.StatusCode, err) + } + + c.Logger.Debugf("created test report record: id=%d, build=%d", tr.GetID(), c.build.GetNumber()) + + return tr, nil +} diff --git a/executor/linux/testattachments.go b/executor/linux/testattachments.go new file mode 100644 index 00000000..2c60c0b2 --- /dev/null +++ b/executor/linux/testattachments.go @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: Apache-2.0 + +package linux + +import ( + "fmt" + "path/filepath" + "strconv" + "time" + + api "github.com/go-vela/server/api/types" +) + +// CreateTestAttachment creates a test attachment record in the database +// after a file has been successfully uploaded to storage. +func (c *client) CreateTestAttachment(fileName, filePath string, size int64, tr *api.TestReport) error { + // extract file extension and type information + fileExt := filepath.Ext(fileName) + + // create object path matching the storage upload format + objectPath := fmt.Sprintf("%s/%s/%s/%s", + c.build.GetRepo().GetOrg(), + c.build.GetRepo().GetName(), + strconv.FormatInt(c.build.GetNumber(), 10), + fileName) + + // create timestamp for record creation + createdAt := time.Now().Unix() + + // build test attachment record + testAttachment := &api.TestAttachment{ + TestReportID: tr.ID, // will be populated by the API based on build context + FileName: &fileName, + ObjectPath: &objectPath, + FileSize: &size, + FileType: &fileExt, + PresignedUrl: nil, // will be generated by the API/storage service + CreatedAt: &createdAt, + } + + // update test attachment in database + ta, resp, err := c.Vela.TestAttachment.Update( + c.build.GetRepo().GetOrg(), + c.build.GetRepo().GetName(), + c.build.GetNumber(), + testAttachment, + ) + if err != nil { + return fmt.Errorf("failed to create test attachment record: build=%d, status=%d, error=%w", + c.build.GetNumber(), resp.StatusCode, err) + } + + c.Logger.Debugf("created test attachment record: id=%d, file=%s", ta.GetID(), fileName) + + return nil +} diff --git a/runtime/docker/test_report.go b/runtime/docker/test_report.go index edee8e95..c6118b71 100644 --- a/runtime/docker/test_report.go +++ b/runtime/docker/test_report.go @@ -59,53 +59,53 @@ func (c *client) execContainerLines(ctx context.Context, containerID, cmd string return lines, nil } -// PollFileNames grabs files name from provided path -// within a container and uploads them to s3 +// PollFileNames searches for files matching the provided patterns within a container. func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, paths []string) ([]string, error) { c.Logger.Tracef("gathering files from container %s", ctn.ID) + if ctn.Image == "" { return nil, nil } - results := make([]string, 0) - for _, p := range paths { - // use find on the container to locate candidates - cmd := fmt.Sprintf("find / -type f -path '*%s' -print", p) - c.Logger.Infof("running: %s", cmd) + var results []string + + for _, pattern := range paths { + // use find command to locate files matching the pattern + cmd := fmt.Sprintf("find / -type f -path '*%s' -print", pattern) + c.Logger.Debugf("searching for files with pattern: %s", pattern) lines, err := c.execContainerLines(ctx, ctn.ID, cmd) if err != nil { - return nil, fmt.Errorf("searching %q: %w", p, err) + return nil, fmt.Errorf("failed to search for pattern %q: %w", pattern, err) } - c.Logger.Tracef("candidates: %v", lines) - for _, raw := range lines { - // 1) strip whitespace (including CR/LF) and normalize the path - fp := filepath.Clean(strings.TrimSpace(raw)) - if fp == "" { + c.Logger.Tracef("found %d candidates for pattern %s", len(lines), pattern) + + // process each found file + for _, line := range lines { + filePath := filepath.Clean(strings.TrimSpace(line)) + if filePath == "" { continue } - // 2) quick ext check - ext := strings.ToLower(filepath.Ext(fp)) + // check if file extension is allowed + ext := strings.ToLower(filepath.Ext(filePath)) if !isAllowedExt(ext) { - c.Logger.Infof("skipping %s (ext %s not allowed)", fp, ext) + c.Logger.Debugf("skipping file %s (extension %s not allowed)", filePath, ext) continue } - // 3) accept - c.Logger.Infof("accepted file: %s", fp) - results = append(results, fp) + c.Logger.Debugf("accepted file: %s", filePath) + results = append(results, filePath) } } - // Add logging for empty results in PollFileNames if len(results) == 0 { - c.Logger.Errorf("PollFileNames found no matching files for paths: %v", paths) - return results, fmt.Errorf("no matching files found for any paths %v", paths) + return results, fmt.Errorf("no matching files found for patterns: %v", paths) } - return results, nil + c.Logger.Infof("found %d files matching patterns", len(results)) + return results, nil } // PollFileContent retrieves the content and size of a file inside a container. From 9c829e5757496a1aea3917ce2e76d51b34e00e32 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Fri, 22 Aug 2025 11:32:37 -0500 Subject: [PATCH 17/63] make clean --- executor/linux/outputs.go | 2 +- executor/linux/testattachments.go | 2 +- executor/local/outputs.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/executor/linux/outputs.go b/executor/linux/outputs.go index 8ee695a9..a74701bd 100644 --- a/executor/linux/outputs.go +++ b/executor/linux/outputs.go @@ -7,9 +7,9 @@ import ( "context" "encoding/base64" "fmt" + "maps" "path/filepath" "strconv" - "maps" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" diff --git a/executor/linux/testattachments.go b/executor/linux/testattachments.go index 2c60c0b2..026fd407 100644 --- a/executor/linux/testattachments.go +++ b/executor/linux/testattachments.go @@ -34,7 +34,7 @@ func (c *client) CreateTestAttachment(fileName, filePath string, size int64, tr ObjectPath: &objectPath, FileSize: &size, FileType: &fileExt, - PresignedUrl: nil, // will be generated by the API/storage service + PresignedURL: nil, // will be generated by the API/storage service CreatedAt: &createdAt, } diff --git a/executor/local/outputs.go b/executor/local/outputs.go index 068437fb..55be4a21 100644 --- a/executor/local/outputs.go +++ b/executor/local/outputs.go @@ -108,7 +108,7 @@ func (o *outputSvc) poll(ctx context.Context, ctn *pipeline.Container) (map[stri "/vela/outputs/base64.env", "/vela/outputs/masked.base64.env", } - + outputMap = make(map[string]string) maskMap = make(map[string]string) ) From 746ba27ed4716fd6d94fdf37c7aabfb566d720c5 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Fri, 22 Aug 2025 12:04:34 -0500 Subject: [PATCH 18/63] point to sdk go branch --- go.mod | 6 +++--- go.sum | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 1ba7cf3c..fed28ef2 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/go-vela/worker -go 1.24.4 +go 1.24.6 require ( github.com/Masterminds/semver/v3 v3.4.0 @@ -8,8 +8,8 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/docker/go-units v0.5.0 github.com/gin-gonic/gin v1.10.1 - github.com/go-vela/sdk-go v0.27.0 - github.com/go-vela/server v0.27.0 + github.com/go-vela/sdk-go v0.27.1-0.20250822165943-9e87fa33539b + github.com/go-vela/server v0.27.1-0.20250821204710-4975ca87a686 github.com/golang-jwt/jwt/v5 v5.3.0 github.com/google/go-cmp v0.7.0 github.com/joho/godotenv v1.5.1 diff --git a/go.sum b/go.sum index 93381b6d..82594c7e 100644 --- a/go.sum +++ b/go.sum @@ -125,10 +125,10 @@ github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/go-vela/sdk-go v0.27.0 h1:nVK58o1n0brl7g8W9Hva+WnlsjCX11b2FGCCuSPkVQg= -github.com/go-vela/sdk-go v0.27.0/go.mod h1:3YyFgyxcvVzqVyrB/ThsPRkNsgf+oobgWFGkPwjLDjI= -github.com/go-vela/server v0.27.0 h1:QUcThEP67UnLvf4YPp7dOPQgRL9MgYI2xYjGops4B5g= -github.com/go-vela/server v0.27.0/go.mod h1:Zlqc4UMaURd1NWTaMy1uw98lKwj2HrCp6BswnL8fpKI= +github.com/go-vela/sdk-go v0.27.1-0.20250822165943-9e87fa33539b h1:nQ06afySLDfReS68mZfLsBs2104FD08/ZNAdTGWrgD4= +github.com/go-vela/sdk-go v0.27.1-0.20250822165943-9e87fa33539b/go.mod h1:wBCxZicyFKGF+IYybDGz4mZodYPYTfgeQSTStSX/GnQ= +github.com/go-vela/server v0.27.1-0.20250821204710-4975ca87a686 h1:o/buvWAibXhrOQIzDoFXQiQ3DpUHZ7cn+4sthFJQa4w= +github.com/go-vela/server v0.27.1-0.20250821204710-4975ca87a686/go.mod h1:+HIDor0dOStt/ROa1Na8IN/p0JO23jJtXBTNDwn2blM= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= From 788af047d8f7f8caf97fd047c3915cfdd4dee389 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Tue, 26 Aug 2025 13:34:51 -0500 Subject: [PATCH 19/63] gomod tidy --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fed28ef2..065cc51b 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/docker/go-units v0.5.0 github.com/gin-gonic/gin v1.10.1 - github.com/go-vela/sdk-go v0.27.1-0.20250822165943-9e87fa33539b + github.com/go-vela/sdk-go v0.27.1-0.20250826182024-d77976452001 github.com/go-vela/server v0.27.1-0.20250821204710-4975ca87a686 github.com/golang-jwt/jwt/v5 v5.3.0 github.com/google/go-cmp v0.7.0 diff --git a/go.sum b/go.sum index 82594c7e..7b5e5a82 100644 --- a/go.sum +++ b/go.sum @@ -125,8 +125,8 @@ github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/go-vela/sdk-go v0.27.1-0.20250822165943-9e87fa33539b h1:nQ06afySLDfReS68mZfLsBs2104FD08/ZNAdTGWrgD4= -github.com/go-vela/sdk-go v0.27.1-0.20250822165943-9e87fa33539b/go.mod h1:wBCxZicyFKGF+IYybDGz4mZodYPYTfgeQSTStSX/GnQ= +github.com/go-vela/sdk-go v0.27.1-0.20250826182024-d77976452001 h1:Lbwg2w8wmsJAyhCZH6pLdHa/TU/gYhrMLOV2/Dn4XSQ= +github.com/go-vela/sdk-go v0.27.1-0.20250826182024-d77976452001/go.mod h1:wBCxZicyFKGF+IYybDGz4mZodYPYTfgeQSTStSX/GnQ= github.com/go-vela/server v0.27.1-0.20250821204710-4975ca87a686 h1:o/buvWAibXhrOQIzDoFXQiQ3DpUHZ7cn+4sthFJQa4w= github.com/go-vela/server v0.27.1-0.20250821204710-4975ca87a686/go.mod h1:+HIDor0dOStt/ROa1Na8IN/p0JO23jJtXBTNDwn2blM= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= From 085a207fbf32666a76c982d5d110da03e854c0f1 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Wed, 27 Aug 2025 10:34:41 -0500 Subject: [PATCH 20/63] fixing linters --- cmd/vela-worker/exec.go | 2 +- cmd/vela-worker/flags.go | 2 +- cmd/vela-worker/run.go | 5 ++-- cmd/vela-worker/start.go | 6 +++++ cmd/vela-worker/worker.go | 2 +- executor/linux/build.go | 6 +++-- executor/linux/build_test.go | 32 ++++++++++++++++++++++++++ executor/linux/linux.go | 3 ++- executor/linux/opts.go | 9 ++++---- executor/linux/outputs.go | 5 ++-- executor/linux/secret.go | 8 +++---- executor/linux/secret_test.go | 1 + executor/local/build.go | 2 +- executor/local/build_test.go | 6 +++++ executor/local/stage_test.go | 1 + executor/setup.go | 4 ++-- executor/setup_test.go | 2 +- internal/context/context_test.go | 2 ++ mock/docker/container.go | 6 ++--- router/middleware/perm/perm_test.go | 6 ++--- runtime/docker/build.go | 6 ++--- runtime/docker/image.go | 4 ++-- runtime/docker/test_report.go | 12 ++++++++++ runtime/docker/test_report_test.go | 2 ++ runtime/kubernetes/build.go | 6 ++--- runtime/kubernetes/build_test.go | 2 +- runtime/kubernetes/container.go | 4 ++-- runtime/kubernetes/pod_tracker.go | 16 ++++++------- runtime/kubernetes/pod_tracker_test.go | 11 +++++---- 29 files changed, 121 insertions(+), 52 deletions(-) diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index 7540e34d..13235566 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -25,7 +25,7 @@ import ( // exec is a helper function to poll the queue // and execute Vela pipelines for the Worker. // -//nolint:nilerr,funlen // ignore returning nil - don't want to crash worker +//nolint:funlen // ignore returning nil - don't want to crash worker func (w *Worker) exec(index int, config *api.Worker) error { var err error diff --git a/cmd/vela-worker/flags.go b/cmd/vela-worker/flags.go index 0dcc3cb6..c4dfbee6 100644 --- a/cmd/vela-worker/flags.go +++ b/cmd/vela-worker/flags.go @@ -5,13 +5,13 @@ package main import ( "context" "fmt" - "github.com/go-vela/server/storage" "strings" "time" "github.com/urfave/cli/v3" "github.com/go-vela/server/queue" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor" "github.com/go-vela/worker/runtime" ) diff --git a/cmd/vela-worker/run.go b/cmd/vela-worker/run.go index 19fde9dd..32a36d26 100644 --- a/cmd/vela-worker/run.go +++ b/cmd/vela-worker/run.go @@ -5,7 +5,6 @@ package main import ( "context" "fmt" - "github.com/go-vela/server/storage" "net/url" "github.com/gin-gonic/gin" @@ -18,6 +17,7 @@ import ( "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" "github.com/go-vela/server/queue" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor" "github.com/go-vela/worker/runtime" ) @@ -105,7 +105,7 @@ func run(ctx context.Context, c *cli.Command) error { // executor configuration Executor: &executor.Setup{ Driver: c.String("executor.driver"), - MaxLogSize: uint(c.Uint("executor.max_log_size")), + MaxLogSize: c.Uint("executor.max_log_size"), LogStreamingTimeout: c.Duration("executor.log_streaming_timeout"), EnforceTrustedRepos: c.Bool("executor.enforce-trusted-repos"), OutputCtn: outputsCtn, @@ -187,5 +187,6 @@ func run(ctx context.Context, c *cli.Command) error { } // start the worker + //nolint: contextcheck return w.Start() } diff --git a/cmd/vela-worker/start.go b/cmd/vela-worker/start.go index 4cf15d5f..58d020bf 100644 --- a/cmd/vela-worker/start.go +++ b/cmd/vela-worker/start.go @@ -47,17 +47,21 @@ func (w *Worker) Start() error { select { case sig := <-signalChannel: logrus.Infof("Received signal: %s", sig) + err := server.Shutdown(ctx) if err != nil { logrus.Error(err) } + done() case <-gctx.Done(): logrus.Info("Closing signal goroutine") + err := server.Shutdown(ctx) if err != nil { logrus.Error(err) } + return gctx.Err() } @@ -67,7 +71,9 @@ func (w *Worker) Start() error { // spawn goroutine for starting the server g.Go(func() error { var err error + logrus.Info("starting worker server") + if tlsCfg != nil { if err := server.ListenAndServeTLS(w.Config.Certificate.Cert, w.Config.Certificate.Key); !errors.Is(err, http.ErrServerClosed) { // log a message indicating the start of the server diff --git a/cmd/vela-worker/worker.go b/cmd/vela-worker/worker.go index 8c4298c0..722883aa 100644 --- a/cmd/vela-worker/worker.go +++ b/cmd/vela-worker/worker.go @@ -3,7 +3,6 @@ package main import ( - "github.com/go-vela/server/storage" "net/url" "sync" "time" @@ -11,6 +10,7 @@ import ( "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/queue" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor" "github.com/go-vela/worker/runtime" ) diff --git a/executor/linux/build.go b/executor/linux/build.go index 66f4332a..c975545d 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -295,7 +295,7 @@ func (c *client) AssembleBuild(ctx context.Context) error { for _, s := range c.pipeline.Stages { // TODO: remove hardcoded reference // - //nolint:goconst // ignore making a constant for now + if s.Name == "init" { continue } @@ -572,12 +572,14 @@ func (c *client) ExecBuild(ctx context.Context) error { testReportCreated = true } + if len(_step.TestReport.Results) != 0 { err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.TestReport.Results, c.build, tr) if err != nil { c.Logger.Errorf("unable to poll files for results: %v", err) } } + if len(_step.TestReport.Attachments) != 0 { err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.TestReport.Attachments, c.build, tr) if err != nil { @@ -734,7 +736,7 @@ func (c *client) StreamBuild(ctx context.Context) error { // into the container right before execution, rather than // during build planning. It is only available for the Docker runtime. // -//nolint:funlen // explanation takes up a lot of lines + func loadLazySecrets(c *client, _step *pipeline.Container) error { _log := new(api.Log) diff --git a/executor/linux/build_test.go b/executor/linux/build_test.go index 56446428..67569219 100644 --- a/executor/linux/build_test.go +++ b/executor/linux/build_test.go @@ -147,6 +147,7 @@ func TestLinux_CreateBuild(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { logger := testLogger.WithFields(logrus.Fields{"test": test.name}) + defer loggerHook.Reset() _pipeline, _, err := compiler. @@ -166,6 +167,7 @@ func TestLinux_CreateBuild(t *testing.T) { switch test.runtime { case constants.DriverKubernetes: _pod := testPodFor(_pipeline) + _runtime, err = kubernetes.NewMock(_pod) if err != nil { t.Errorf("unable to create kubernetes runtime engine: %v", err) @@ -204,16 +206,19 @@ func TestLinux_CreateBuild(t *testing.T) { } loggedError := false + for _, logEntry := range loggerHook.AllEntries() { // Many errors during StreamBuild get logged and ignored. // So, Make sure there are no errors logged during StreamBuild. if logEntry.Level == logrus.ErrorLevel { loggedError = true + if !test.logError { t.Errorf("%s StreamBuild for %s logged an Error: %v", test.name, test.pipeline, logEntry.Message) } } } + if test.logError && !loggedError { t.Errorf("%s StreamBuild for %s did not log an Error but should have", test.name, test.pipeline) } @@ -330,6 +335,7 @@ func TestLinux_PlanBuild(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { logger := testLogger.WithFields(logrus.Fields{"test": test.name}) + defer loggerHook.Reset() _pipeline, _, err := compiler. @@ -349,6 +355,7 @@ func TestLinux_PlanBuild(t *testing.T) { switch test.runtime { case constants.DriverKubernetes: _pod := testPodFor(_pipeline) + _runtime, err = kubernetes.NewMock(_pod) if err != nil { t.Errorf("unable to create kubernetes runtime engine: %v", err) @@ -392,16 +399,19 @@ func TestLinux_PlanBuild(t *testing.T) { } loggedError := false + for _, logEntry := range loggerHook.AllEntries() { // Many errors during StreamBuild get logged and ignored. // So, Make sure there are no errors logged during StreamBuild. if logEntry.Level == logrus.ErrorLevel { loggedError = true + if !test.logError { t.Errorf("%s StreamBuild for %s logged an Error: %v", test.name, test.pipeline, logEntry.Message) } } } + if test.logError && !loggedError { t.Errorf("%s StreamBuild for %s did not log an Error but should have", test.name, test.pipeline) } @@ -619,6 +629,7 @@ func TestLinux_AssembleBuild(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { logger := testLogger.WithFields(logrus.Fields{"test": test.name}) + defer loggerHook.Reset() _pipeline, _, err := compiler. @@ -706,16 +717,19 @@ func TestLinux_AssembleBuild(t *testing.T) { } loggedError := false + for _, logEntry := range loggerHook.AllEntries() { // Many errors during StreamBuild get logged and ignored. // So, Make sure there are no errors logged during StreamBuild. if logEntry.Level == logrus.ErrorLevel { loggedError = true + if !test.logError { t.Errorf("%s StreamBuild for %s logged an Error: %v", test.name, test.pipeline, logEntry.Message) } } } + if test.logError && !loggedError { t.Errorf("%s StreamBuild for %s did not log an Error but should have", test.name, test.pipeline) } @@ -860,6 +874,7 @@ func TestLinux_ExecBuild(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { logger := testLogger.WithFields(logrus.Fields{"test": test.name}) + defer loggerHook.Reset() _pipeline, _, err := compiler. @@ -882,6 +897,7 @@ func TestLinux_ExecBuild(t *testing.T) { switch test.runtime { case constants.DriverKubernetes: _pod = testPodFor(_pipeline) + _runtime, err = kubernetes.NewMock(_pod) if err != nil { t.Errorf("unable to create kubernetes runtime engine: %v", err) @@ -965,6 +981,7 @@ func TestLinux_ExecBuild(t *testing.T) { percents := []int{0, 0, 50, 100} lastIndex := len(percents) - 1 + for index, stepsCompletedPercent := range percents { if index == 0 || index == lastIndex { stepsRunningCount = 0 @@ -1002,16 +1019,19 @@ func TestLinux_ExecBuild(t *testing.T) { } loggedError := false + for _, logEntry := range loggerHook.AllEntries() { // Many errors during StreamBuild get logged and ignored. // So, Make sure there are no errors logged during StreamBuild. if logEntry.Level == logrus.ErrorLevel { loggedError = true + if !test.logError { t.Errorf("%s StreamBuild for %s logged an Error: %v", test.name, test.pipeline, logEntry.Message) } } } + if test.logError && !loggedError { t.Errorf("%s StreamBuild for %s did not log an Error but should have", test.name, test.pipeline) } @@ -1509,6 +1529,7 @@ func TestLinux_StreamBuild(t *testing.T) { streamRequests := make(chan message.StreamRequest) logger := testLogger.WithFields(logrus.Fields{"test": test.name}) + defer loggerHook.Reset() _pipeline, _, err := compiler. @@ -1528,6 +1549,7 @@ func TestLinux_StreamBuild(t *testing.T) { switch test.runtime { case constants.DriverKubernetes: _pod := testPodFor(_pipeline) + _runtime, err = kubernetes.NewMock(_pod) if err != nil { t.Errorf("unable to create kubernetes runtime engine: %v", err) @@ -1575,10 +1597,12 @@ func TestLinux_StreamBuild(t *testing.T) { // imitate build getting canceled or otherwise finishing before ExecBuild gets called. done() } + if test.earlyExecExit { // imitate a failure after ExecBuild starts and before it sends a StreamRequest. close(streamRequests) } + if test.earlyBuildDone || test.earlyExecExit { return } @@ -1621,16 +1645,19 @@ func TestLinux_StreamBuild(t *testing.T) { } loggedError := false + for _, logEntry := range loggerHook.AllEntries() { // Many errors during StreamBuild get logged and ignored. // So, Make sure there are no errors logged during StreamBuild. if logEntry.Level == logrus.ErrorLevel { loggedError = true + if !test.logError { t.Errorf("%s StreamBuild for %s logged an Error: %v", test.name, test.pipeline, logEntry.Message) } } } + if test.logError && !loggedError { t.Errorf("%s StreamBuild for %s did not log an Error but should have", test.name, test.pipeline) } @@ -1789,6 +1816,7 @@ func TestLinux_DestroyBuild(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { logger := testLogger.WithFields(logrus.Fields{"test": test.name}) + defer loggerHook.Reset() _pipeline, _, err := compiler. @@ -1808,6 +1836,7 @@ func TestLinux_DestroyBuild(t *testing.T) { switch test.runtime { case constants.DriverKubernetes: _pod := testPodFor(_pipeline) + _runtime, err = kubernetes.NewMock(_pod) if err != nil { t.Errorf("unable to create kubernetes runtime engine: %v", err) @@ -1860,6 +1889,7 @@ func TestLinux_DestroyBuild(t *testing.T) { } loggedError := false + for _, logEntry := range loggerHook.AllEntries() { // Many errors during StreamBuild get logged and ignored. // So, Make sure there are no errors logged during StreamBuild. @@ -1879,11 +1909,13 @@ func TestLinux_DestroyBuild(t *testing.T) { } loggedError = true + if !test.logError { t.Errorf("%s StreamBuild for %s logged an Error: %v", test.name, test.pipeline, logEntry.Message) } } } + if test.logError && !loggedError { t.Errorf("%s StreamBuild for %s did not log an Error but should have", test.name, test.pipeline) } diff --git a/executor/linux/linux.go b/executor/linux/linux.go index 188b08e0..c9e8bd48 100644 --- a/executor/linux/linux.go +++ b/executor/linux/linux.go @@ -8,13 +8,14 @@ import ( "sync" "time" + "github.com/sirupsen/logrus" + "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime" - "github.com/sirupsen/logrus" ) type ( diff --git a/executor/linux/opts.go b/executor/linux/opts.go index 397e106c..b5c3c008 100644 --- a/executor/linux/opts.go +++ b/executor/linux/opts.go @@ -4,7 +4,6 @@ package linux import ( "fmt" - "github.com/go-vela/server/storage" "time" "github.com/sirupsen/logrus" @@ -12,6 +11,7 @@ import ( "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime" ) @@ -223,13 +223,14 @@ func WithStorage(s *storage.Setup) Opt { // set the storage in the client var err error + c.Storage, err = storage.New(s) if err != nil { - return fmt.Errorf("unable to create storage: %v", err) - + return fmt.Errorf("unable to create storage: %w", err) } + if c.Storage == nil { - return fmt.Errorf("empty storage setup: %v", err) + return fmt.Errorf("empty storage setup: %w", err) } return nil diff --git a/executor/linux/outputs.go b/executor/linux/outputs.go index a74701bd..9ba867e4 100644 --- a/executor/linux/outputs.go +++ b/executor/linux/outputs.go @@ -11,10 +11,11 @@ import ( "path/filepath" "strconv" - api "github.com/go-vela/server/api/types" - "github.com/go-vela/server/compiler/types/pipeline" envparse "github.com/hashicorp/go-envparse" "github.com/sirupsen/logrus" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/compiler/types/pipeline" ) // outputSvc handles communication with the outputs container during the build. diff --git a/executor/linux/secret.go b/executor/linux/secret.go index 3a4fe988..a98a8909 100644 --- a/executor/linux/secret.go +++ b/executor/linux/secret.go @@ -214,7 +214,7 @@ func (s *secretSvc) pull(secret *pipeline.Secret) (*api.Secret, error) { // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#SecretService.Get _secret, _, err = s.client.Vela.Secret.Get(secret.Engine, secret.Type, org, "*", key) if err != nil { - return nil, fmt.Errorf("%s: %w", ErrUnableToRetrieve, err) + return nil, fmt.Errorf("%w: %w", ErrUnableToRetrieve, err) } secret.Value = _secret.GetValue() @@ -231,7 +231,7 @@ func (s *secretSvc) pull(secret *pipeline.Secret) (*api.Secret, error) { // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#SecretService.Get _secret, _, err = s.client.Vela.Secret.Get(secret.Engine, secret.Type, org, repo, key) if err != nil { - return nil, fmt.Errorf("%s: %w", ErrUnableToRetrieve, err) + return nil, fmt.Errorf("%w: %w", ErrUnableToRetrieve, err) } secret.Value = _secret.GetValue() @@ -248,7 +248,7 @@ func (s *secretSvc) pull(secret *pipeline.Secret) (*api.Secret, error) { // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#SecretService.Get _secret, _, err = s.client.Vela.Secret.Get(secret.Engine, secret.Type, org, team, key) if err != nil { - return nil, fmt.Errorf("%s: %w", ErrUnableToRetrieve, err) + return nil, fmt.Errorf("%w: %w", ErrUnableToRetrieve, err) } secret.Value = _secret.GetValue() @@ -366,7 +366,7 @@ func escapeNewlineSecrets(m map[string]*api.Secret) { for i, secret := range m { // only double-escape secrets that have been manually escaped if !strings.Contains(secret.GetValue(), "\\\\n") { - s := strings.Replace(secret.GetValue(), "\\n", "\\\n", -1) + s := strings.ReplaceAll(secret.GetValue(), "\\n", "\\\n") m[i].Value = &s } } diff --git a/executor/linux/secret_test.go b/executor/linux/secret_test.go index ba0008b0..6a09e0a8 100644 --- a/executor/linux/secret_test.go +++ b/executor/linux/secret_test.go @@ -438,6 +438,7 @@ func TestLinux_Secret_exec(t *testing.T) { switch test.runtime { case constants.DriverKubernetes: _pod := testPodFor(p) + _runtime, err = kubernetes.NewMock(_pod) if err != nil { t.Errorf("unable to create kubernetes runtime engine: %v", err) diff --git a/executor/local/build.go b/executor/local/build.go index 796cb131..1e6b3037 100644 --- a/executor/local/build.go +++ b/executor/local/build.go @@ -194,7 +194,7 @@ func (c *client) AssembleBuild(ctx context.Context) error { for _, _stage := range c.pipeline.Stages { // TODO: remove hardcoded reference // - //nolint:goconst // ignore making a constant for now + if _stage.Name == "init" { continue } diff --git a/executor/local/build_test.go b/executor/local/build_test.go index 71b98039..9f523301 100644 --- a/executor/local/build_test.go +++ b/executor/local/build_test.go @@ -25,6 +25,7 @@ func TestLocal_CreateBuild(t *testing.T) { Usage: "doc", }, } + compiler, err := native.FromCLICommand(context.Background(), cmd) if err != nil { t.Errorf("unable to create compiler engine: %v", err) @@ -110,6 +111,7 @@ func TestLocal_PlanBuild(t *testing.T) { Usage: "doc", }, } + compiler, err := native.FromCLICommand(context.Background(), cmd) if err != nil { t.Errorf("unable to create compiler engine: %v", err) @@ -201,6 +203,7 @@ func TestLocal_AssembleBuild(t *testing.T) { Usage: "doc", }, } + compiler, err := native.FromCLICommand(context.Background(), cmd) if err != nil { t.Errorf("unable to create compiler engine: %v", err) @@ -326,6 +329,7 @@ func TestLocal_ExecBuild(t *testing.T) { Usage: "doc", }, } + compiler, err := native.FromCLICommand(context.Background(), cmd) if err != nil { t.Errorf("unable to create compiler engine: %v", err) @@ -436,6 +440,7 @@ func TestLocal_StreamBuild(t *testing.T) { Usage: "doc", }, } + compiler, err := native.FromCLICommand(context.Background(), cmd) if err != nil { t.Errorf("unable to create compiler engine: %v", err) @@ -658,6 +663,7 @@ func TestLocal_DestroyBuild(t *testing.T) { Usage: "doc", }, } + compiler, err := native.FromCLICommand(context.Background(), cmd) if err != nil { t.Errorf("unable to create compiler engine: %v", err) diff --git a/executor/local/stage_test.go b/executor/local/stage_test.go index dadae0c1..a8a4b3c2 100644 --- a/executor/local/stage_test.go +++ b/executor/local/stage_test.go @@ -29,6 +29,7 @@ func TestLocal_CreateStage(t *testing.T) { Usage: "doc", }, } + compiler, err := native.FromCLICommand(context.Background(), cmd) if err != nil { t.Errorf("unable to create compiler from CLI context: %v", err) diff --git a/executor/setup.go b/executor/setup.go index f28e95fa..5314b37c 100644 --- a/executor/setup.go +++ b/executor/setup.go @@ -4,7 +4,6 @@ package executor import ( "fmt" - "github.com/go-vela/server/storage" "strings" "time" @@ -14,6 +13,7 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor/linux" "github.com/go-vela/worker/executor/local" "github.com/go-vela/worker/runtime" @@ -174,7 +174,7 @@ func (s *Setup) Validate() error { } // check if the storage client is provided - if &s.Storage == nil { + if s.Storage == nil { return fmt.Errorf("no storage client provided in setup") } diff --git a/executor/setup_test.go b/executor/setup_test.go index 8bcba8c4..10cec265 100644 --- a/executor/setup_test.go +++ b/executor/setup_test.go @@ -3,7 +3,6 @@ package executor import ( - "github.com/go-vela/server/storage" "net/http/httptest" "testing" "time" @@ -15,6 +14,7 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/constants" "github.com/go-vela/server/mock/server" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor/linux" "github.com/go-vela/worker/executor/local" "github.com/go-vela/worker/runtime/docker" diff --git a/internal/context/context_test.go b/internal/context/context_test.go index 4e1ba5d3..4c89c5d6 100644 --- a/internal/context/context_test.go +++ b/internal/context/context_test.go @@ -129,6 +129,7 @@ func TestWithDelayedCancelPropagation(t *testing.T) { if d := ctx.Done(); d == nil { t.Errorf("ctx.Done() == %v want non-nil", d) } + if e := ctx.Err(); e != nil { t.Errorf("ctx.Err() == %v want nil", e) } @@ -145,6 +146,7 @@ func TestWithDelayedCancelPropagation(t *testing.T) { testCancelPropagated(ctx, "WithDelayedCancelPropagation", t) time.Sleep(shortDuration) + lastLogEntry := loggerHook.LastEntry() if lastLogEntry.Message != test.lastLogMessage { t.Errorf("unexpected last log entry: want = %s ; got = %s", test.lastLogMessage, lastLogEntry.Message) diff --git a/mock/docker/container.go b/mock/docker/container.go index e1df4360..2a9dbc0e 100644 --- a/mock/docker/container.go +++ b/mock/docker/container.go @@ -61,7 +61,7 @@ func (c *ContainerService) ContainerCreate(ctx context.Context, config *containe !strings.Contains(ctn, "ignorenotfound") { return container.CreateResponse{}, //nolint:stylecheck // messsage is capitalized to match Docker messages - errdefs.NotFound(fmt.Errorf("Error: No such container: %s", ctn)) + errdefs.NotFound(fmt.Errorf("error: No such container: %s", ctn)) } // check if the container is not-found and @@ -70,7 +70,7 @@ func (c *ContainerService) ContainerCreate(ctx context.Context, config *containe !strings.Contains(ctn, "ignore-not-found") { return container.CreateResponse{}, //nolint:stylecheck // messsage is capitalized to match Docker messages - errdefs.NotFound(fmt.Errorf("Error: No such container: %s", ctn)) + errdefs.NotFound(fmt.Errorf("error: No such container: %s", ctn)) } // check if the image is not found @@ -79,7 +79,7 @@ func (c *ContainerService) ContainerCreate(ctx context.Context, config *containe return container.CreateResponse{}, errdefs.NotFound( //nolint:stylecheck // messsage is capitalized to match Docker messages - fmt.Errorf("Error response from daemon: manifest for %s not found: manifest unknown", config.Image), + fmt.Errorf("error response from daemon: manifest for %s not found: manifest unknown", config.Image), ) } diff --git a/router/middleware/perm/perm_test.go b/router/middleware/perm/perm_test.go index 41cb0cdc..5ef3c7a8 100644 --- a/router/middleware/perm/perm_test.go +++ b/router/middleware/perm/perm_test.go @@ -23,7 +23,7 @@ func TestPerm_MustServer_ValidateToken200(t *testing.T) { workerCtx, workerEngine := gin.CreateTestContext(workerResp) // fake request made to the worker router - workerCtx.Request, _ = http.NewRequest(http.MethodGet, "/build/cancel", nil) + workerCtx.Request, _ = http.NewRequestWithContext(t.Context(), http.MethodGet, "/build/cancel", nil) workerCtx.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tkn)) // setup mock server router @@ -71,7 +71,7 @@ func TestPerm_MustServer_ValidateToken401(t *testing.T) { workerCtx, workerEngine := gin.CreateTestContext(workerResp) // fake request made to the worker router - workerCtx.Request, _ = http.NewRequest(http.MethodGet, "/build/cancel", nil) + workerCtx.Request, _ = http.NewRequestWithContext(t.Context(), http.MethodGet, "/build/cancel", nil) workerCtx.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tkn)) // setup mock server router @@ -119,7 +119,7 @@ func TestPerm_MustServer_ValidateToken404(t *testing.T) { workerCtx, workerEngine := gin.CreateTestContext(workerResp) // fake request made to the worker router - workerCtx.Request, _ = http.NewRequest(http.MethodGet, "/build/cancel", nil) + workerCtx.Request, _ = http.NewRequestWithContext(t.Context(), http.MethodGet, "/build/cancel", nil) workerCtx.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tkn)) // setup mock server router diff --git a/runtime/docker/build.go b/runtime/docker/build.go index dfe3571d..4d8512c8 100644 --- a/runtime/docker/build.go +++ b/runtime/docker/build.go @@ -10,7 +10,7 @@ import ( // InspectBuild displays details about the pod for the init step. // This is a no-op for docker. -func (c *client) InspectBuild(ctx context.Context, b *pipeline.Build) ([]byte, error) { +func (c *client) InspectBuild(_ context.Context, b *pipeline.Build) ([]byte, error) { c.Logger.Tracef("no-op: inspecting build for pipeline %s", b.ID) return []byte{}, nil @@ -18,7 +18,7 @@ func (c *client) InspectBuild(ctx context.Context, b *pipeline.Build) ([]byte, e // SetupBuild prepares the pipeline build. // This is a no-op for docker. -func (c *client) SetupBuild(ctx context.Context, b *pipeline.Build) error { +func (c *client) SetupBuild(_ context.Context, b *pipeline.Build) error { c.Logger.Tracef("no-op: setting up for build %s", b.ID) return nil @@ -26,7 +26,7 @@ func (c *client) SetupBuild(ctx context.Context, b *pipeline.Build) error { // StreamBuild initializes log/event streaming for build. // This is a no-op for docker. -func (c *client) StreamBuild(ctx context.Context, b *pipeline.Build) error { +func (c *client) StreamBuild(_ context.Context, b *pipeline.Build) error { c.Logger.Tracef("no-op: streaming build %s", b.ID) return nil diff --git a/runtime/docker/image.go b/runtime/docker/image.go index 67888873..d01b6864 100644 --- a/runtime/docker/image.go +++ b/runtime/docker/image.go @@ -87,8 +87,8 @@ func (c *client) InspectImage(ctx context.Context, ctn *pipeline.Container) ([]b // send API call to inspect the image // - // https://pkg.go.dev/github.com/docker/docker/client#Client.ImageInspectWithRaw - i, _, err := c.Docker.ImageInspectWithRaw(ctx, _image) + // https://pkg.go.dev/github.com/docker/docker/client#Client.ImageInspect + i, err := c.Docker.ImageInspect(ctx, _image) if err != nil { return output, err } diff --git a/runtime/docker/test_report.go b/runtime/docker/test_report.go index c6118b71..424b3725 100644 --- a/runtime/docker/test_report.go +++ b/runtime/docker/test_report.go @@ -26,6 +26,7 @@ func isAllowedExt(ext string) bool { return true } } + return false } @@ -38,24 +39,30 @@ func (c *client) execContainerLines(ctx context.Context, containerID, cmd string AttachStdout: true, AttachStderr: true, } + resp, err := c.Docker.ContainerExecCreate(ctx, containerID, execConfig) if err != nil { return nil, fmt.Errorf("create exec: %w", err) } + attach, err := c.Docker.ContainerExecAttach(ctx, resp.ID, dockerContainerTypes.ExecAttachOptions{}) if err != nil { return nil, fmt.Errorf("attach exec: %w", err) } + defer attach.Close() var outBuf, errBuf bytes.Buffer if _, err := stdcopy.StdCopy(&outBuf, &errBuf, attach.Reader); err != nil { return nil, fmt.Errorf("copy exec output: %w", err) } + if errBuf.Len() > 0 { return nil, fmt.Errorf("exec error: %s", errBuf.String()) } + lines := strings.Split(strings.TrimSpace(outBuf.String()), "\n") + return lines, nil } @@ -105,6 +112,7 @@ func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, pat } c.Logger.Infof("found %d files matching patterns", len(results)) + return results, nil } @@ -116,6 +124,7 @@ func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, p // return an empty reader instead of nil return bytes.NewReader(nil), 0, fmt.Errorf("empty container image") } + cmd := []string{"sh", "-c", fmt.Sprintf("base64 %s", path)} execConfig := dockerContainerTypes.ExecOptions{ Cmd: cmd, @@ -125,11 +134,13 @@ func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, p } c.Logger.Infof("executing command for content: %v", execConfig.Cmd) + execID, err := c.Docker.ContainerExecCreate(ctx, ctn.ID, execConfig) if err != nil { c.Logger.Debugf("PollFileContent exec-create failed for %q: %v", path, err) return nil, 0, fmt.Errorf("failed to create exec instance: %w", err) } + resp, err := c.Docker.ContainerExecAttach(ctx, execID.ID, dockerContainerTypes.ExecAttachOptions{}) if err != nil { c.Logger.Debugf("PollFileContent exec-attach failed for %q: %v", path, err) @@ -155,6 +166,7 @@ func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, p if outputStderr.Len() > 0 { return nil, 0, fmt.Errorf("error: %s", outputStderr.String()) } + data := outputStdout.Bytes() // Add logging for empty data in PollFileContent diff --git a/runtime/docker/test_report_test.go b/runtime/docker/test_report_test.go index 84ef352d..85df4ded 100644 --- a/runtime/docker/test_report_test.go +++ b/runtime/docker/test_report_test.go @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 + package docker // diff --git a/runtime/kubernetes/build.go b/runtime/kubernetes/build.go index c1fb23b9..754fd35c 100644 --- a/runtime/kubernetes/build.go +++ b/runtime/kubernetes/build.go @@ -225,7 +225,7 @@ func (c *client) AssembleBuild(ctx context.Context, b *pipeline.Build) error { // remnants get deleted. c.createdPod = true - c.Logger.Infof("creating pod %s", c.Pod.ObjectMeta.Name) + c.Logger.Infof("creating pod %s", c.Pod.Name) // send API call to create the pod // // https://pkg.go.dev/k8s.io/client-go/kubernetes/typed/core/v1#PodInterface @@ -276,11 +276,11 @@ func (c *client) RemoveBuild(ctx context.Context, b *pipeline.Build) error { PropagationPolicy: &policy, } - c.Logger.Infof("removing pod %s", c.Pod.ObjectMeta.Name) + c.Logger.Infof("removing pod %s", c.Pod.Name) // send API call to delete the pod err := c.Kubernetes.CoreV1(). Pods(c.config.Namespace). - Delete(ctx, c.Pod.ObjectMeta.Name, opts) + Delete(ctx, c.Pod.Name, opts) if err != nil { return err } diff --git a/runtime/kubernetes/build_test.go b/runtime/kubernetes/build_test.go index 70cd3cb0..69c18b5b 100644 --- a/runtime/kubernetes/build_test.go +++ b/runtime/kubernetes/build_test.go @@ -311,7 +311,7 @@ func TestKubernetes_SetupBuild(t *testing.T) { } // make sure that worker-defined labels are set and cannot be overridden by PipelinePodsTemplate - if pipelineLabel, ok := _engine.Pod.ObjectMeta.Labels["pipeline"]; !ok { + if pipelineLabel, ok := _engine.Pod.Labels["pipeline"]; !ok { t.Errorf("Pod is missing the pipeline label: %v", _engine.Pod.ObjectMeta) } else if pipelineLabel != test.pipeline.ID { t.Errorf("Pod's pipeline label is %v, want %v", pipelineLabel, test.pipeline.ID) diff --git a/runtime/kubernetes/container.go b/runtime/kubernetes/container.go index 5c78d093..a039bb92 100644 --- a/runtime/kubernetes/container.go +++ b/runtime/kubernetes/container.go @@ -27,7 +27,7 @@ func (c *client) InspectContainer(ctx context.Context, ctn *pipeline.Container) // get the pod from the local cache, which the Informer keeps up-to-date pod, err := c.PodTracker.PodLister. Pods(c.config.Namespace). - Get(c.Pod.ObjectMeta.Name) + Get(c.Pod.Name) if err != nil { return err } @@ -110,7 +110,7 @@ func (c *client) RunContainer(ctx context.Context, ctn *pipeline.Container, _ *p // https://pkg.go.dev/k8s.io/client-go/kubernetes/typed/core/v1#PodInterface _, err = c.Kubernetes.CoreV1().Pods(c.config.Namespace).Patch( ctx, - c.Pod.ObjectMeta.Name, + c.Pod.Name, types.StrategicMergePatchType, []byte(fmt.Sprintf(imagePatch, ctn.ID, _image)), metav1.PatchOptions{}, diff --git a/runtime/kubernetes/pod_tracker.go b/runtime/kubernetes/pod_tracker.go index a94dc2e6..2ce654d5 100644 --- a/runtime/kubernetes/pod_tracker.go +++ b/runtime/kubernetes/pod_tracker.go @@ -183,8 +183,8 @@ func newPodTracker(log *logrus.Entry, clientset kubernetes.Interface, pod *v1.Po return nil, fmt.Errorf("newPodTracker expected a pod, got nil") } - trackedPod := pod.ObjectMeta.Namespace + "/" + pod.ObjectMeta.Name - if pod.ObjectMeta.Name == "" || pod.ObjectMeta.Namespace == "" { + trackedPod := pod.Namespace + "/" + pod.Name + if pod.Name == "" || pod.Namespace == "" { return nil, fmt.Errorf("newPodTracker expects pod to have Name and Namespace, got %s", trackedPod) } @@ -194,7 +194,7 @@ func newPodTracker(log *logrus.Entry, clientset kubernetes.Interface, pod *v1.Po selector, err := labels.NewRequirement( "pipeline", selection.Equals, - []string{fields.EscapeValue(pod.ObjectMeta.Name)}, + []string{fields.EscapeValue(pod.Name)}, ) if err != nil { return nil, err @@ -204,7 +204,7 @@ func newPodTracker(log *logrus.Entry, clientset kubernetes.Interface, pod *v1.Po informerFactory := kubeinformers.NewSharedInformerFactoryWithOptions( clientset, defaultResync, - kubeinformers.WithNamespace(pod.ObjectMeta.Namespace), + kubeinformers.WithNamespace(pod.Namespace), kubeinformers.WithTweakListOptions(func(listOptions *metav1.ListOptions) { listOptions.LabelSelector = selector.String() }), @@ -235,12 +235,12 @@ func newPodTracker(log *logrus.Entry, clientset kubernetes.Interface, pod *v1.Po // mockPodTracker returns a new podTracker with the given pod pre-loaded in the cache. func mockPodTracker(log *logrus.Entry, clientset kubernetes.Interface, pod *v1.Pod) (*podTracker, error) { // Make sure test pods are valid before passing to PodTracker (ie support &v1.Pod{}). - if pod.ObjectMeta.Name == "" { - pod.ObjectMeta.Name = "test-pod" + if pod.Name == "" { + pod.Name = "test-pod" } - if pod.ObjectMeta.Namespace == "" { - pod.ObjectMeta.Namespace = "test" + if pod.Namespace == "" { + pod.Namespace = "test" } tracker, err := newPodTracker(log, clientset, pod, 0*time.Second) diff --git a/runtime/kubernetes/pod_tracker_test.go b/runtime/kubernetes/pod_tracker_test.go index 99f159fd..8ec7ec31 100644 --- a/runtime/kubernetes/pod_tracker_test.go +++ b/runtime/kubernetes/pod_tracker_test.go @@ -52,8 +52,8 @@ func TestNewPodTracker(t *testing.T) { pod: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "github-octocat-1-for-some-odd-reason-this-name-is-way-too-long-and-will-cause-an-error", - Namespace: _pod.ObjectMeta.Namespace, - Labels: _pod.ObjectMeta.Labels, + Namespace: _pod.Namespace, + Labels: _pod.Labels, }, TypeMeta: _pod.TypeMeta, Spec: _pod.Spec, @@ -181,7 +181,7 @@ func Test_podTracker_HandlePodAdd(t *testing.T) { }, } for _, test := range tests { - t.Run(test.name, func(t *testing.T) { + t.Run(test.name, func(_ *testing.T) { p := &podTracker{ Logger: logger, TrackedPod: test.trackedPod, @@ -255,7 +255,7 @@ func Test_podTracker_HandlePodUpdate(t *testing.T) { }, } for _, test := range tests { - t.Run(test.name, func(t *testing.T) { + t.Run(test.name, func(_ *testing.T) { p := &podTracker{ Logger: logger, TrackedPod: test.trackedPod, @@ -324,7 +324,7 @@ func Test_podTracker_HandlePodDelete(t *testing.T) { }, } for _, test := range tests { - t.Run(test.name, func(t *testing.T) { + t.Run(test.name, func(_ *testing.T) { p := &podTracker{ Logger: logger, TrackedPod: test.trackedPod, @@ -370,6 +370,7 @@ func Test_podTracker_Stop(t *testing.T) { if test.started { tracker.Start(context.Background()) } + tracker.Stop() }) } From e9a7ced84a263b3bdb084f973791eb1112767ae6 Mon Sep 17 00:00:00 2001 From: Tim Huynh Date: Wed, 27 Aug 2025 10:36:53 -0500 Subject: [PATCH 21/63] Potential fix for code scanning alert no. 5: Clear-text logging of sensitive information Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- cmd/vela-worker/operate.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/vela-worker/operate.go b/cmd/vela-worker/operate.go index e102eab2..e88da8af 100644 --- a/cmd/vela-worker/operate.go +++ b/cmd/vela-worker/operate.go @@ -96,7 +96,6 @@ func (w *Worker) operate(ctx context.Context) error { logrus.Trace("access key: ", w.Config.Executor.Storage.AccessKey) // set secret key in storage config w.Config.Executor.Storage.SecretKey = stCreds.GetSecretKey() - logrus.Trace("secret key: ", w.Config.Executor.Storage.SecretKey) // set bucket name in storage config w.Config.Executor.Storage.Bucket = stCreds.GetStorageBucket() From fe1107856d3baf39eea4cce41602ca5b27cfc0e7 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Thu, 28 Aug 2025 14:22:50 -0500 Subject: [PATCH 22/63] add and fix tests --- cmd/vela-worker/run.go | 2 +- executor/executor_test.go | 15 +++++ executor/linux/build.go | 2 +- executor/linux/build_test.go | 73 +++++++++++++++++++++ executor/linux/stage_test.go | 49 ++++++++++++++ executor/local/build.go | 5 ++ executor/local/build_test.go | 73 +++++++++++++++++++++ executor/local/local.go | 2 + executor/local/opts.go | 26 ++++++++ executor/local/stage_test.go | 13 ++++ executor/setup.go | 1 + executor/setup_test.go | 25 +++++++ router/middleware/executor/executor_test.go | 25 +++++++ 13 files changed, 309 insertions(+), 2 deletions(-) diff --git a/cmd/vela-worker/run.go b/cmd/vela-worker/run.go index 32a36d26..e99caff8 100644 --- a/cmd/vela-worker/run.go +++ b/cmd/vela-worker/run.go @@ -187,6 +187,6 @@ func run(ctx context.Context, c *cli.Command) error { } // start the worker - //nolint: contextcheck + //nolint: contextcheck // not using ctx yet return w.Start() } diff --git a/executor/executor_test.go b/executor/executor_test.go index 48b0c6a5..910a021f 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/gin-gonic/gin" + "github.com/go-vela/server/storage" "github.com/google/go-cmp/cmp" "github.com/go-vela/sdk-go/vela" @@ -37,6 +38,17 @@ func TestExecutor_New(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + _linux, err := linux.New( linux.WithBuild(_build), linux.WithHostname("localhost"), @@ -45,6 +57,7 @@ func TestExecutor_New(t *testing.T) { linux.WithRuntime(_runtime), linux.WithVelaClient(_client), linux.WithVersion("v1.0.0"), + linux.WithStorage(_storage), ) if err != nil { t.Errorf("unable to create linux engine: %v", err) @@ -95,6 +108,7 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", + Storage: _storage, }, want: _linux, equal: linux.Equal, @@ -109,6 +123,7 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", + Storage: _storage, }, want: _local, equal: local.Equal, diff --git a/executor/linux/build.go b/executor/linux/build.go index c975545d..a2e0694c 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -30,7 +30,7 @@ func (c *client) CreateBuild(ctx context.Context) error { // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Snapshot defer func() { build.Snapshot(c.build, c.Vela, c.err, c.Logger) }() // Check if storage client is initialized - // and if storage is enable + // and if storage is enabled if c.Storage == nil { return fmt.Errorf("storage client is not initialized") } diff --git a/executor/linux/build_test.go b/executor/linux/build_test.go index 67569219..2771c060 100644 --- a/executor/linux/build_test.go +++ b/executor/linux/build_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/go-vela/server/storage" "github.com/sirupsen/logrus" logrusTest "github.com/sirupsen/logrus/hooks/test" "github.com/urfave/cli/v3" @@ -179,6 +180,17 @@ func TestLinux_CreateBuild(t *testing.T) { } } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + _engine, err := New( WithLogger(logger), WithBuild(test.build), @@ -186,6 +198,7 @@ func TestLinux_CreateBuild(t *testing.T) { WithRuntime(_runtime), WithVelaClient(_client), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -252,6 +265,17 @@ func TestLinux_PlanBuild(t *testing.T) { t.Errorf("unable to create Vela API client: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + tests := []struct { name string failure bool @@ -373,6 +397,7 @@ func TestLinux_PlanBuild(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithVelaClient(_client), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -448,6 +473,17 @@ func TestLinux_AssembleBuild(t *testing.T) { streamRequests, done := message.MockStreamRequestsWithCancel(context.Background()) defer done() + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + tests := []struct { name string failure bool @@ -667,6 +703,7 @@ func TestLinux_AssembleBuild(t *testing.T) { WithVelaClient(_client), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -763,6 +800,17 @@ func TestLinux_ExecBuild(t *testing.T) { t.Errorf("unable to create Vela API client: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + tests := []struct { name string failure bool @@ -920,6 +968,7 @@ func TestLinux_ExecBuild(t *testing.T) { WithVelaClient(_client), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -1065,6 +1114,17 @@ func TestLinux_StreamBuild(t *testing.T) { t.Errorf("unable to create Vela API client: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + type planFuncType = func(context.Context, *pipeline.Container) error // planNothing is a planFuncType that does nothing @@ -1569,6 +1629,7 @@ func TestLinux_StreamBuild(t *testing.T) { WithLogStreamingTimeout(1*time.Second), WithVelaClient(_client), withStreamRequests(streamRequests), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -1691,6 +1752,17 @@ func TestLinux_DestroyBuild(t *testing.T) { t.Errorf("unable to create Vela API client: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + tests := []struct { name string failure bool @@ -1855,6 +1927,7 @@ func TestLinux_DestroyBuild(t *testing.T) { WithRuntime(_runtime), WithVelaClient(_client), WithOutputCtn(testOutputsCtn()), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) diff --git a/executor/linux/stage_test.go b/executor/linux/stage_test.go index 043f6cda..311d37df 100644 --- a/executor/linux/stage_test.go +++ b/executor/linux/stage_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/gin-gonic/gin" + "github.com/go-vela/server/storage" "github.com/urfave/cli/v3" "github.com/go-vela/sdk-go/vela" @@ -66,6 +67,17 @@ func TestLinux_CreateStage(t *testing.T) { t.Errorf("unable to create kubernetes runtime engine: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + // setup tests tests := []struct { name string @@ -171,6 +183,7 @@ func TestLinux_CreateStage(t *testing.T) { WithPipeline(_pipeline), WithRuntime(test.runtime), WithVelaClient(_client), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -224,6 +237,17 @@ func TestLinux_PlanStage(t *testing.T) { t.Errorf("unable to create kubernetes runtime engine: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + dockerTestMap := new(sync.Map) dockerTestMap.Store("foo", make(chan error, 1)) @@ -394,6 +418,7 @@ func TestLinux_PlanStage(t *testing.T) { WithPipeline(new(pipeline.Build)), WithRuntime(test.runtime), WithVelaClient(_client), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -444,6 +469,17 @@ func TestLinux_ExecStage(t *testing.T) { streamRequests, done := message.MockStreamRequestsWithCancel(context.Background()) defer done() + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + // setup tests tests := []struct { name string @@ -583,6 +619,7 @@ func TestLinux_ExecStage(t *testing.T) { WithVelaClient(_client), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -628,6 +665,17 @@ func TestLinux_DestroyStage(t *testing.T) { t.Errorf("unable to create kubernetes runtime engine: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + // setup tests tests := []struct { name string @@ -683,6 +731,7 @@ func TestLinux_DestroyStage(t *testing.T) { WithPipeline(new(pipeline.Build)), WithRuntime(test.runtime), WithVelaClient(_client), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) diff --git a/executor/local/build.go b/executor/local/build.go index 1e6b3037..b2a0ee7d 100644 --- a/executor/local/build.go +++ b/executor/local/build.go @@ -23,6 +23,11 @@ func (c *client) CreateBuild(ctx context.Context) error { // // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Snapshot defer func() { build.Snapshot(c.build, nil, c.err, nil) }() + // Check if storage client is initialized + // and if storage is enabled + if c.Storage == nil { + return fmt.Errorf("storage client is not initialized") + } // update the build fields c.build.SetStatus(constants.StatusRunning) diff --git a/executor/local/build_test.go b/executor/local/build_test.go index 9f523301..d5ee5991 100644 --- a/executor/local/build_test.go +++ b/executor/local/build_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/go-vela/server/storage" "github.com/urfave/cli/v3" "github.com/go-vela/server/compiler/native" @@ -38,6 +39,17 @@ func TestLocal_CreateBuild(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + tests := []struct { name string failure bool @@ -79,6 +91,7 @@ func TestLocal_CreateBuild(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -124,6 +137,17 @@ func TestLocal_PlanBuild(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + tests := []struct { name string failure bool @@ -165,6 +189,7 @@ func TestLocal_PlanBuild(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -216,6 +241,17 @@ func TestLocal_AssembleBuild(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + streamRequests, done := message.MockStreamRequestsWithCancel(context.Background()) defer done() @@ -291,6 +327,7 @@ func TestLocal_AssembleBuild(t *testing.T) { WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -342,6 +379,17 @@ func TestLocal_ExecBuild(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + streamRequests, done := message.MockStreamRequestsWithCancel(context.Background()) defer done() @@ -402,6 +450,7 @@ func TestLocal_ExecBuild(t *testing.T) { WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -453,6 +502,17 @@ func TestLocal_StreamBuild(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + type planFuncType = func(context.Context, *pipeline.Container) error // planNothing is a planFuncType that does nothing @@ -605,6 +665,7 @@ func TestLocal_StreamBuild(t *testing.T) { WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -676,6 +737,17 @@ func TestLocal_DestroyBuild(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + tests := []struct { name string failure bool @@ -732,6 +804,7 @@ func TestLocal_DestroyBuild(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) diff --git a/executor/local/local.go b/executor/local/local.go index dca23942..bc0304f1 100644 --- a/executor/local/local.go +++ b/executor/local/local.go @@ -10,6 +10,7 @@ import ( "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime" ) @@ -22,6 +23,7 @@ type ( Hostname string Version string OutputCtn *pipeline.Container + Storage storage.Storage // private fields init *pipeline.Container diff --git a/executor/local/opts.go b/executor/local/opts.go index 61660ed7..e67e82a8 100644 --- a/executor/local/opts.go +++ b/executor/local/opts.go @@ -9,6 +9,7 @@ import ( "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime" ) @@ -140,3 +141,28 @@ func withStreamRequests(s chan message.StreamRequest) Opt { return nil } } + +// WithStorage sets the storage in the executor client for Linux. +func WithStorage(s *storage.Setup) Opt { + return func(c *client) error { + + // check if the storage provided is empty + if &s == nil { + return fmt.Errorf("empty storage setup provided") + } + + // set the storage in the client + var err error + + c.Storage, err = storage.New(s) + if err != nil { + return fmt.Errorf("unable to create storage: %w", err) + } + + if c.Storage == nil { + return fmt.Errorf("empty storage setup: %w", err) + } + + return nil + } +} diff --git a/executor/local/stage_test.go b/executor/local/stage_test.go index a8a4b3c2..8c25ae7f 100644 --- a/executor/local/stage_test.go +++ b/executor/local/stage_test.go @@ -8,6 +8,7 @@ import ( "sync" "testing" + "github.com/go-vela/server/storage" "github.com/urfave/cli/v3" "github.com/go-vela/server/compiler/native" @@ -51,6 +52,17 @@ func TestLocal_CreateStage(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + // setup tests tests := []struct { name string @@ -103,6 +115,7 @@ func TestLocal_CreateStage(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), + WithStorage(_storage), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) diff --git a/executor/setup.go b/executor/setup.go index 5314b37c..148afaf0 100644 --- a/executor/setup.go +++ b/executor/setup.go @@ -115,6 +115,7 @@ func (s *Setup) Local() (Engine, error) { local.WithVersion(s.Version), local.WithMockStdout(s.Mock), local.WithOutputCtn(s.OutputCtn), + local.WithStorage(s.Storage), ) } diff --git a/executor/setup_test.go b/executor/setup_test.go index 10cec265..2ecb6c96 100644 --- a/executor/setup_test.go +++ b/executor/setup_test.go @@ -137,6 +137,17 @@ func TestExecutor_Setup_Local(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + want, err := local.New( local.WithBuild(_build), local.WithHostname("localhost"), @@ -144,6 +155,7 @@ func TestExecutor_Setup_Local(t *testing.T) { local.WithRuntime(_runtime), local.WithVelaClient(_client), local.WithVersion("v1.0.0"), + local.WithStorage(_storage), ) if err != nil { t.Errorf("unable to create local engine: %v", err) @@ -157,6 +169,7 @@ func TestExecutor_Setup_Local(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", + Storage: _storage, } // run test @@ -222,6 +235,17 @@ func TestExecutor_Setup_Validate(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + _emptyOwnerBuild := new(api.Build) _emptyOwnerBuild.SetRepo(new(api.Repo)) @@ -242,6 +266,7 @@ func TestExecutor_Setup_Validate(t *testing.T) { MaxLogSize: 2097152, Pipeline: _pipeline, Runtime: _runtime, + Storage: _storage, }, failure: false, }, diff --git a/router/middleware/executor/executor_test.go b/router/middleware/executor/executor_test.go index e522af22..103f2c75 100644 --- a/router/middleware/executor/executor_test.go +++ b/router/middleware/executor/executor_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/gin-gonic/gin" + "github.com/go-vela/server/storage" "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" @@ -33,6 +34,17 @@ func TestExecutor_Retrieve(t *testing.T) { _build := new(api.Build) _build.SetRepo(_repo) + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + want, err := executor.New(&executor.Setup{ Driver: constants.DriverLinux, MaxLogSize: 2097152, @@ -40,6 +52,7 @@ func TestExecutor_Retrieve(t *testing.T) { Runtime: _runtime, Build: _build, Pipeline: new(pipeline.Build), + Storage: _storage, }) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -72,6 +85,17 @@ func TestExecutor_Establish(t *testing.T) { _build := new(api.Build) _build.SetRepo(_repo) + _storage := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "ad", + SecretKey: "asd", + Bucket: "vela", + Region: "", + Secure: false, + } + want, err := executor.New(&executor.Setup{ Driver: constants.DriverLinux, MaxLogSize: 2097152, @@ -79,6 +103,7 @@ func TestExecutor_Establish(t *testing.T) { Runtime: _runtime, Build: _build, Pipeline: new(pipeline.Build), + Storage: _storage, }) if err != nil { t.Errorf("unable to create executor engine: %v", err) From 14d86452685df0ae7f8029eaf6f685b2a562505b Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Thu, 28 Aug 2025 14:34:20 -0500 Subject: [PATCH 23/63] golanci run fix --- cmd/vela-worker/run.go | 2 +- executor/executor_test.go | 2 +- executor/linux/build.go | 1 - executor/linux/build_test.go | 2 +- executor/linux/stage_test.go | 2 +- executor/local/build.go | 1 - executor/local/build_test.go | 2 +- executor/local/opts.go | 1 - executor/local/stage_test.go | 2 +- router/middleware/executor/executor_test.go | 2 +- 10 files changed, 7 insertions(+), 10 deletions(-) diff --git a/cmd/vela-worker/run.go b/cmd/vela-worker/run.go index e99caff8..0f940791 100644 --- a/cmd/vela-worker/run.go +++ b/cmd/vela-worker/run.go @@ -97,7 +97,7 @@ func run(ctx context.Context, c *cli.Command) error { }, // build configuration Build: &Build{ - Limit: int(c.Int("build.limit")), + Limit: c.Int("build.limit"), Timeout: c.Duration("build.timeout"), }, // build configuration diff --git a/executor/executor_test.go b/executor/executor_test.go index 910a021f..78ed6d13 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -8,7 +8,6 @@ import ( "testing" "github.com/gin-gonic/gin" - "github.com/go-vela/server/storage" "github.com/google/go-cmp/cmp" "github.com/go-vela/sdk-go/vela" @@ -17,6 +16,7 @@ import ( "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" "github.com/go-vela/server/mock/server" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor/linux" "github.com/go-vela/worker/executor/local" "github.com/go-vela/worker/runtime/docker" diff --git a/executor/linux/build.go b/executor/linux/build.go index a2e0694c..470eeb6d 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -295,7 +295,6 @@ func (c *client) AssembleBuild(ctx context.Context) error { for _, s := range c.pipeline.Stages { // TODO: remove hardcoded reference // - if s.Name == "init" { continue } diff --git a/executor/linux/build_test.go b/executor/linux/build_test.go index 2771c060..fe82f600 100644 --- a/executor/linux/build_test.go +++ b/executor/linux/build_test.go @@ -10,7 +10,6 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/go-vela/server/storage" "github.com/sirupsen/logrus" logrusTest "github.com/sirupsen/logrus/hooks/test" "github.com/urfave/cli/v3" @@ -22,6 +21,7 @@ import ( "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" "github.com/go-vela/server/mock/server" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime" "github.com/go-vela/worker/runtime/docker" diff --git a/executor/linux/stage_test.go b/executor/linux/stage_test.go index 311d37df..9b5b433a 100644 --- a/executor/linux/stage_test.go +++ b/executor/linux/stage_test.go @@ -10,13 +10,13 @@ import ( "testing" "github.com/gin-gonic/gin" - "github.com/go-vela/server/storage" "github.com/urfave/cli/v3" "github.com/go-vela/sdk-go/vela" "github.com/go-vela/server/compiler/native" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/mock/server" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime" "github.com/go-vela/worker/runtime/docker" diff --git a/executor/local/build.go b/executor/local/build.go index b2a0ee7d..792418bf 100644 --- a/executor/local/build.go +++ b/executor/local/build.go @@ -199,7 +199,6 @@ func (c *client) AssembleBuild(ctx context.Context) error { for _, _stage := range c.pipeline.Stages { // TODO: remove hardcoded reference // - if _stage.Name == "init" { continue } diff --git a/executor/local/build_test.go b/executor/local/build_test.go index d5ee5991..a20f7179 100644 --- a/executor/local/build_test.go +++ b/executor/local/build_test.go @@ -7,11 +7,11 @@ import ( "testing" "time" - "github.com/go-vela/server/storage" "github.com/urfave/cli/v3" "github.com/go-vela/server/compiler/native" "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime/docker" ) diff --git a/executor/local/opts.go b/executor/local/opts.go index e67e82a8..ed4f1eee 100644 --- a/executor/local/opts.go +++ b/executor/local/opts.go @@ -145,7 +145,6 @@ func withStreamRequests(s chan message.StreamRequest) Opt { // WithStorage sets the storage in the executor client for Linux. func WithStorage(s *storage.Setup) Opt { return func(c *client) error { - // check if the storage provided is empty if &s == nil { return fmt.Errorf("empty storage setup provided") diff --git a/executor/local/stage_test.go b/executor/local/stage_test.go index 8c25ae7f..b3f57dce 100644 --- a/executor/local/stage_test.go +++ b/executor/local/stage_test.go @@ -8,11 +8,11 @@ import ( "sync" "testing" - "github.com/go-vela/server/storage" "github.com/urfave/cli/v3" "github.com/go-vela/server/compiler/native" "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime/docker" ) diff --git a/router/middleware/executor/executor_test.go b/router/middleware/executor/executor_test.go index 103f2c75..11f69269 100644 --- a/router/middleware/executor/executor_test.go +++ b/router/middleware/executor/executor_test.go @@ -9,12 +9,12 @@ import ( "testing" "github.com/gin-gonic/gin" - "github.com/go-vela/server/storage" "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor" "github.com/go-vela/worker/runtime/docker" ) From 5e087705de43a10d420c729a07e09aa6e461301f Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Mon, 29 Sep 2025 16:09:30 -0500 Subject: [PATCH 24/63] point to server branch --- go.mod | 8 ++++---- go.sum | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index e9679d9c..5e376e8c 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,8 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/docker/go-units v0.5.0 github.com/gin-gonic/gin v1.10.1 - github.com/go-vela/sdk-go v0.27.1 - github.com/go-vela/server v0.27.1 + github.com/go-vela/sdk-go v0.27.2-0.20250929210753-8aa86d4e80fa + github.com/go-vela/server v0.27.3-0.20250929162825-a9dc2bb27fbd github.com/golang-jwt/jwt/v5 v5.3.0 github.com/google/go-cmp v0.7.0 github.com/joho/godotenv v1.5.1 @@ -36,9 +36,9 @@ require ( github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/go-ini/ini v1.67.0 // indirect github.com/expr-lang/expr v1.17.6 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/google/go-github/v74 v74.0.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect @@ -60,8 +60,8 @@ require ( github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/minio-go/v7 v7.0.89 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rs/xid v1.6.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/uptrace/opentelemetry-go-extra/otelgorm v0.3.2 // indirect github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect diff --git a/go.sum b/go.sum index 0e4955db..9175122e 100644 --- a/go.sum +++ b/go.sum @@ -125,10 +125,10 @@ github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/go-vela/sdk-go v0.27.1 h1:SMNENVv+iqoXZg0NSptR9U5jT9BM7a6z7EihszCVs4c= -github.com/go-vela/sdk-go v0.27.1/go.mod h1:g7cKpm7RTsIe3R4c5SeZ5fFdk+LXFQ50oJ4YGsnaB+s= -github.com/go-vela/server v0.27.1 h1:rWEUfTJoy05q0qyJf61OK4LqvLwAzjQMg8dMDVbKpZk= -github.com/go-vela/server v0.27.1/go.mod h1:TxRPsPhkfHwbIpaurSMQgLwQFrIu+uZPQPLo5M8s2oc= +github.com/go-vela/sdk-go v0.27.2-0.20250929210753-8aa86d4e80fa h1:WicAh+R8C1/9WguXVr9wShMRiyTcrg12LOo+qMod/Sg= +github.com/go-vela/sdk-go v0.27.2-0.20250929210753-8aa86d4e80fa/go.mod h1:NujXkANYx1vkTfWeJXzk1lNJ4tzjnZF+0Pa1hqBrSTQ= +github.com/go-vela/server v0.27.3-0.20250929162825-a9dc2bb27fbd h1:YAbUQDQy1Z0OkpaxeaLd1vQ+yttqNP0G8fMiIr6SIbE= +github.com/go-vela/server v0.27.3-0.20250929162825-a9dc2bb27fbd/go.mod h1:ziaGq1cWl+y6C+Vz+FxvuiqZnDtzRneqDqTuzTra8VE= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= From 319412fbd02560731bffd0b6f3205f9d215e7011 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Tue, 14 Oct 2025 15:47:15 -0500 Subject: [PATCH 25/63] non working code --- cmd/vela-worker/exec.go | 74 ++++++++---- cmd/vela-worker/operate.go | 43 ++++--- cmd/vela-worker/run.go | 9 +- executor/executor_test.go | 31 ++++- executor/linux/build.go | 120 +++++++++++++++----- executor/linux/build_test.go | 41 ++++++- executor/linux/opts.go | 12 +- executor/linux/stage.go | 2 +- executor/linux/stage_test.go | 28 ++++- executor/local/build.go | 8 +- executor/local/build_test.go | 38 ++++++- executor/local/opts.go | 12 +- executor/local/stage.go | 2 +- executor/local/stage_test.go | 6 +- executor/setup.go | 44 ++++--- executor/setup_test.go | 25 +++- go.mod | 5 + go.sum | 4 - internal/step/skip.go | 9 +- internal/step/skip_test.go | 25 +++- router/middleware/executor/executor_test.go | 12 +- 21 files changed, 405 insertions(+), 145 deletions(-) diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index 13235566..c34c8e0d 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/go-vela/server/storage" "github.com/sirupsen/logrus" "github.com/go-vela/sdk-go/vela" @@ -28,6 +29,8 @@ import ( //nolint:funlen // ignore returning nil - don't want to crash worker func (w *Worker) exec(index int, config *api.Worker) error { var err error + var execStorage *storage.Storage + var _executor executor.Engine // setup the version v := version.New() @@ -156,8 +159,12 @@ func (w *Worker) exec(index int, config *api.Worker) error { execOutputCtn := *w.Config.Executor.OutputCtn execOutputCtn.ID = fmt.Sprintf("outputs_%s", p.ID) + logrus.Debugf("storage config: %+v", w.Config.Storage.Enable) // dereference configured storage config and set the storage config for the executor - execStorage := *w.Config.Executor.Storage + if w.Config.Storage.Enable { + logrus.Debugf("executor storage is enabled") + execStorage = w.Config.Executor.Storage + } // create logger with extra metadata // @@ -233,26 +240,51 @@ func (w *Worker) exec(index int, config *api.Worker) error { return err } - // setup the executor - // - // https://pkg.go.dev/github.com/go-vela/worker/executor#New - _executor, err := executor.New(&executor.Setup{ - Logger: logger, - Mock: w.Config.Mock, - Driver: w.Config.Executor.Driver, - MaxLogSize: w.Config.Executor.MaxLogSize, - LogStreamingTimeout: w.Config.Executor.LogStreamingTimeout, - EnforceTrustedRepos: w.Config.Executor.EnforceTrustedRepos, - PrivilegedImages: w.Config.Runtime.PrivilegedImages, - Client: execBuildClient, - Hostname: w.Config.API.Address.Hostname(), - Runtime: w.Runtime, - Build: item.Build, - Pipeline: p.Sanitize(w.Config.Runtime.Driver), - Version: v.Semantic(), - OutputCtn: &execOutputCtn, - Storage: &execStorage, - }) + if w.Config.Storage.Enable { + logrus.Debugf("executor storage is enabled") + logrus.Debugf("storage config: %+v", execStorage) + // setup the executor + // + // https://pkg.go.dev/github.com/go-vela/worker/executor#New + _executor, err = executor.New(&executor.Setup{ + Logger: logger, + Mock: w.Config.Mock, + Driver: w.Config.Executor.Driver, + MaxLogSize: w.Config.Executor.MaxLogSize, + LogStreamingTimeout: w.Config.Executor.LogStreamingTimeout, + EnforceTrustedRepos: w.Config.Executor.EnforceTrustedRepos, + PrivilegedImages: w.Config.Runtime.PrivilegedImages, + Client: execBuildClient, + Hostname: w.Config.API.Address.Hostname(), + Runtime: w.Runtime, + Build: item.Build, + Pipeline: p.Sanitize(w.Config.Runtime.Driver), + Version: v.Semantic(), + OutputCtn: &execOutputCtn, + Storage: execStorage, + }) + } else { + logrus.Debugf("executor storage is disabled") + // setup the executor + // + // https://pkg.go.dev/github.com/go-vela/worker/executor#New + _executor, err = executor.New(&executor.Setup{ + Logger: logger, + Mock: w.Config.Mock, + Driver: w.Config.Executor.Driver, + MaxLogSize: w.Config.Executor.MaxLogSize, + LogStreamingTimeout: w.Config.Executor.LogStreamingTimeout, + EnforceTrustedRepos: w.Config.Executor.EnforceTrustedRepos, + PrivilegedImages: w.Config.Runtime.PrivilegedImages, + Client: execBuildClient, + Hostname: w.Config.API.Address.Hostname(), + Runtime: w.Runtime, + Build: item.Build, + Pipeline: p.Sanitize(w.Config.Runtime.Driver), + Version: v.Semantic(), + OutputCtn: &execOutputCtn, + }) + } // add the executor to the worker w.Executors[index] = _executor diff --git a/cmd/vela-worker/operate.go b/cmd/vela-worker/operate.go index e88da8af..c819dfe9 100644 --- a/cmd/vela-worker/operate.go +++ b/cmd/vela-worker/operate.go @@ -81,25 +81,40 @@ func (w *Worker) operate(ctx context.Context) error { // fetching queue credentials using registration token stCreds, _, err := w.VelaClient.Storage.GetInfo() if err != nil { - logrus.Trace("error getting storage creds") + logrus.Tracef("error getting storage creds: %v", err) return err } + w.Config.Storage.Enable = stCreds.GetEnabled() - // if an address was given at start up, use that — else use what is returned from server - if len(w.Config.Executor.Storage.Endpoint) == 0 { - w.Config.Executor.Storage.Endpoint = stCreds.GetStorageAddress() - logrus.Trace("storage address: ", w.Config.Executor.Storage.Driver) - } + logrus.Trace("Storage enabled: ", w.Config.Storage.Enable) - // set access key in storage config - w.Config.Executor.Storage.AccessKey = stCreds.GetAccessKey() - logrus.Trace("access key: ", w.Config.Executor.Storage.AccessKey) - // set secret key in storage config - w.Config.Executor.Storage.SecretKey = stCreds.GetSecretKey() + if w.Config.Storage.Enable { + logrus.Trace("storage enabled") + // if an address was given at start up, use that — else use what is returned from server + if len(w.Config.Storage.Endpoint) == 0 { + w.Config.Storage.Endpoint = stCreds.GetStorageAddress() + logrus.Trace("storage address: ", w.Config.Storage.Driver) + } + + // set access key in storage config + w.Config.Storage.AccessKey = stCreds.GetAccessKey() + logrus.Trace("access key: ", w.Config.Storage.AccessKey) + // set secret key in storage config + w.Config.Storage.SecretKey = stCreds.GetSecretKey() + + // set bucket name in storage config + w.Config.Storage.Bucket = stCreds.GetStorageBucket() + logrus.Trace("bucket name: ", w.Config.Storage.Bucket) + } else { + logrus.Trace("storage not enabled") + // storage disabled; nothing to validate + w.Config.Storage.Driver = "" + w.Config.Storage.Endpoint = "" + w.Config.Storage.AccessKey = "" + w.Config.Storage.SecretKey = "" + w.Config.Storage.Bucket = "" + } - // set bucket name in storage config - w.Config.Executor.Storage.Bucket = stCreds.GetStorageBucket() - logrus.Trace("bucket name: ", w.Config.Executor.Storage.Bucket) // spawn goroutine for phoning home executors.Go(func() error { // five second ticker for signal handling diff --git a/cmd/vela-worker/run.go b/cmd/vela-worker/run.go index 0f940791..bfda637f 100644 --- a/cmd/vela-worker/run.go +++ b/cmd/vela-worker/run.go @@ -8,6 +8,7 @@ import ( "net/url" "github.com/gin-gonic/gin" + "github.com/go-vela/server/storage" "github.com/sirupsen/logrus" "github.com/urfave/cli/v3" @@ -17,7 +18,6 @@ import ( "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" "github.com/go-vela/server/queue" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor" "github.com/go-vela/worker/runtime" ) @@ -109,13 +109,6 @@ func run(ctx context.Context, c *cli.Command) error { LogStreamingTimeout: c.Duration("executor.log_streaming_timeout"), EnforceTrustedRepos: c.Bool("executor.enforce-trusted-repos"), OutputCtn: outputsCtn, - Storage: &storage.Setup{ - Driver: c.String("storage.driver"), - Endpoint: c.String("storage.endpoint.name"), - AccessKey: c.String("storage.access.key"), - SecretKey: c.String("storage.secret.key"), - Bucket: c.String("storage.bucket.name"), - }, }, // logger configuration Logger: &Logger{ diff --git a/executor/executor_test.go b/executor/executor_test.go index 78ed6d13..c295b927 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -38,7 +38,7 @@ func TestExecutor_New(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } - _storage := &storage.Setup{ + _storageT := &storage.Setup{ Enable: true, Driver: "minio", Endpoint: "http://localhost:9000", @@ -48,6 +48,25 @@ func TestExecutor_New(t *testing.T) { Region: "", Secure: false, } + _sT, err := storage.New(_storageT) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } + + _storageF := &storage.Setup{ + Enable: false, + Driver: "", + Endpoint: "", + AccessKey: "", + SecretKey: "", + Bucket: "", + Region: "", + Secure: false, + } + _sF, err := storage.New(_storageF) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } _linux, err := linux.New( linux.WithBuild(_build), @@ -57,7 +76,7 @@ func TestExecutor_New(t *testing.T) { linux.WithRuntime(_runtime), linux.WithVelaClient(_client), linux.WithVersion("v1.0.0"), - linux.WithStorage(_storage), + linux.WithStorage(&_sT), ) if err != nil { t.Errorf("unable to create linux engine: %v", err) @@ -93,6 +112,7 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", + Storage: &_sF, }, want: nil, equal: reflect.DeepEqual, @@ -108,7 +128,7 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: _storage, + Storage: &_sT, }, want: _linux, equal: linux.Equal, @@ -123,7 +143,7 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: _storage, + Storage: &_sT, }, want: _local, equal: local.Equal, @@ -138,6 +158,7 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", + Storage: &_sT, }, want: nil, equal: reflect.DeepEqual, @@ -152,6 +173,7 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", + Storage: &_sT, }, want: nil, equal: reflect.DeepEqual, @@ -166,6 +188,7 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", + Storage: &_sT, }, want: nil, equal: reflect.DeepEqual, diff --git a/executor/linux/build.go b/executor/linux/build.go index 470eeb6d..9220b6ad 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -31,9 +31,9 @@ func (c *client) CreateBuild(ctx context.Context) error { defer func() { build.Snapshot(c.build, c.Vela, c.err, c.Logger) }() // Check if storage client is initialized // and if storage is enabled - if c.Storage == nil { - return fmt.Errorf("storage client is not initialized") - } + //if c.Storage == nil && c.Storage.StorageEnable() { + // return fmt.Errorf("storage client is not initialized") + //} // update the build fields c.build.SetStatus(constants.StatusRunning) @@ -517,7 +517,7 @@ func (c *client) ExecBuild(ctx context.Context) error { // check if the step should be skipped // // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Skip - skip, err := step.Skip(_step, c.build, c.build.GetStatus()) + skip, err := step.Skip(_step, c.build, c.build.GetStatus(), c.Storage) if err != nil { return fmt.Errorf("unable to plan step: %w", c.err) } @@ -526,6 +526,64 @@ func (c *client) ExecBuild(ctx context.Context) error { continue } + // Check if this step has test_report and storage is disabled + //if !_step.TestReport.Empty() && c.Storage == nil { + // c.Logger.Infof("skipping %s step: storage is disabled but test_report is defined", _step.Name) + // + // //// Load step model + // //stepData, err := step.Load(_step, &c.steps) + // //if err != nil { + // // return fmt.Errorf("unable to load step: %w", err) + // //} + // // + // //// Load or create logs for this step + // ////stepLog, err := step.LoadLogs(_step, &c.stepLogs) + // ////if err != nil { + // //// return fmt.Errorf("unable to load step logs: %w", err) + // ////} + // // + // //// Ensure timestamps + // //now := time.Now().UTC().Unix() + // //if stepData.GetStarted() == 0 { + // // stepData.SetStarted(now) + // //} + // // + // //stepData.SetStatus(constants.StatusError) + // //stepData.SetExitCode(0) + // //stepData.SetFinished(now) + // + // // send API call to update the step + // // + // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#StepService.Update + // //_tsstep, _, err := c.Vela.Step.Update(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), stepData) + // //if err != nil { + // // return err + // //} + // // + // //// send API call to capture the step log + // //// + // //// https://pkg.go.dev/github.com/go-vela/sdk-go/vela#LogService.GetStep + // //_log, _, err := c.Vela.Log.GetStep(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), _tsstep.GetNumber()) + // //if err != nil { + // // return err + // //} + // //_log.AppendData([]byte("Storage is disabled, contact Vela Admins\n")) + // // + // //// add a step log to a map + // //c.stepLogs.Store(_step.ID, _log) + // //stepLog.AppendData([]byte("Storage is disabled, contact Vela Admins\n")) + // //stepLog.SetData([]byte("Storage is disabled, contact Vela Admins\n")) + // //// Upload logs so UI can display the message + // //if _, err := c.Vela.Log. + // // UpdateStep(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), *stepData.Number, stepLog); err != nil { + // // c.Logger.Errorf("unable to upload skipped step logs: %v", err) + // //} + // // Upload step status + // //step.Upload(_step, c.build, c.Vela, c.Logger, stepData) + // + // continue + //} + // add netrc to secrets for masking in logs sec := &pipeline.StepSecret{ Target: "VELA_NETRC_PASSWORD", @@ -556,36 +614,42 @@ func (c *client) ExecBuild(ctx context.Context) error { // logic for polling files only if the test-report step is present // iterate through the steps in the build - if !_step.TestReport.Empty() { - c.Logger.Debug("creating test report record in database") - // send API call to update the test report - // - // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#TestReportService.Add - // TODO: .Add should be .Update - // TODO: handle somewhere if multiple test report keys exist in pipeline - if !testReportCreated { - tr, c.err = c.CreateTestReport() - if c.err != nil { - return fmt.Errorf("unable to create test report: %w", c.err) - } - - testReportCreated = true + // TODO: API to return if storage is enabled + //if c.Storage == nil && _step.TestReport.Empty() || c.Storage == nil && !_step.TestReport.Empty() { + // c.Logger.Infof("storage disabled, skipping test report for %s step", _step.Name) + // // skip if no storage client + // // but test report is defined in step + // continue + //} else if !_step.TestReport.Empty() && c.Storage != nil { + c.Logger.Debug("creating test report record in database") + // send API call to update the test report + // + // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#TestReportService.Add + // TODO: .Add should be .Update + // TODO: handle somewhere if multiple test report keys exist in pipeline + if !testReportCreated { + tr, c.err = c.CreateTestReport() + if c.err != nil { + return fmt.Errorf("unable to create test report: %w", c.err) } - if len(_step.TestReport.Results) != 0 { - err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.TestReport.Results, c.build, tr) - if err != nil { - c.Logger.Errorf("unable to poll files for results: %v", err) - } + testReportCreated = true + } + + if len(_step.TestReport.Results) != 0 { + err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.TestReport.Results, c.build, tr) + if err != nil { + c.Logger.Errorf("unable to poll files for results: %v", err) } + } - if len(_step.TestReport.Attachments) != 0 { - err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.TestReport.Attachments, c.build, tr) - if err != nil { - c.Logger.Errorf("unable to poll files for attachments: %v", err) - } + if len(_step.TestReport.Attachments) != 0 { + err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.TestReport.Attachments, c.build, tr) + if err != nil { + c.Logger.Errorf("unable to poll files for attachments: %v", err) } } + //} // perform any substitution on dynamic variables err = _step.Substitute() diff --git a/executor/linux/build_test.go b/executor/linux/build_test.go index fe82f600..49bbe2e2 100644 --- a/executor/linux/build_test.go +++ b/executor/linux/build_test.go @@ -191,6 +191,11 @@ func TestLinux_CreateBuild(t *testing.T) { Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } + _engine, err := New( WithLogger(logger), WithBuild(test.build), @@ -198,7 +203,7 @@ func TestLinux_CreateBuild(t *testing.T) { WithRuntime(_runtime), WithVelaClient(_client), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -275,6 +280,10 @@ func TestLinux_PlanBuild(t *testing.T) { Region: "", Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } tests := []struct { name string @@ -397,7 +406,7 @@ func TestLinux_PlanBuild(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithVelaClient(_client), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -484,6 +493,11 @@ func TestLinux_AssembleBuild(t *testing.T) { Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } + tests := []struct { name string failure bool @@ -703,7 +717,7 @@ func TestLinux_AssembleBuild(t *testing.T) { WithVelaClient(_client), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -811,6 +825,11 @@ func TestLinux_ExecBuild(t *testing.T) { Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } + tests := []struct { name string failure bool @@ -968,7 +987,7 @@ func TestLinux_ExecBuild(t *testing.T) { WithVelaClient(_client), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -1125,6 +1144,11 @@ func TestLinux_StreamBuild(t *testing.T) { Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } + type planFuncType = func(context.Context, *pipeline.Container) error // planNothing is a planFuncType that does nothing @@ -1629,7 +1653,7 @@ func TestLinux_StreamBuild(t *testing.T) { WithLogStreamingTimeout(1*time.Second), WithVelaClient(_client), withStreamRequests(streamRequests), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -1763,6 +1787,11 @@ func TestLinux_DestroyBuild(t *testing.T) { Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } + tests := []struct { name string failure bool @@ -1927,7 +1956,7 @@ func TestLinux_DestroyBuild(t *testing.T) { WithRuntime(_runtime), WithVelaClient(_client), WithOutputCtn(testOutputsCtn()), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) diff --git a/executor/linux/opts.go b/executor/linux/opts.go index b5c3c008..0599e8b1 100644 --- a/executor/linux/opts.go +++ b/executor/linux/opts.go @@ -212,7 +212,7 @@ func withStreamRequests(s chan message.StreamRequest) Opt { } // WithStorage sets the storage in the executor client for Linux. -func WithStorage(s *storage.Setup) Opt { +func WithStorage(s *storage.Storage) Opt { return func(c *client) error { c.Logger.Trace("configuring storage in linux executor client") @@ -221,16 +221,10 @@ func WithStorage(s *storage.Setup) Opt { return fmt.Errorf("empty storage setup provided") } - // set the storage in the client - var err error - - c.Storage, err = storage.New(s) - if err != nil { - return fmt.Errorf("unable to create storage: %w", err) - } + c.Storage = *s if c.Storage == nil { - return fmt.Errorf("empty storage setup: %w", err) + return fmt.Errorf("empty storage setup: %v", s) } return nil diff --git a/executor/linux/stage.go b/executor/linux/stage.go index 8d67ca94..20b25982 100644 --- a/executor/linux/stage.go +++ b/executor/linux/stage.go @@ -132,7 +132,7 @@ func (c *client) ExecStage(ctx context.Context, s *pipeline.Stage, m *sync.Map) // check if the step should be skipped // // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Skip - skip, err := step.Skip(_step, c.build, useStatus) + skip, err := step.Skip(_step, c.build, useStatus, c.Storage) if err != nil { return fmt.Errorf("unable to plan step: %w", c.err) } diff --git a/executor/linux/stage_test.go b/executor/linux/stage_test.go index 9b5b433a..7aed2171 100644 --- a/executor/linux/stage_test.go +++ b/executor/linux/stage_test.go @@ -78,6 +78,11 @@ func TestLinux_CreateStage(t *testing.T) { Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } + // setup tests tests := []struct { name string @@ -183,7 +188,7 @@ func TestLinux_CreateStage(t *testing.T) { WithPipeline(_pipeline), WithRuntime(test.runtime), WithVelaClient(_client), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -248,6 +253,11 @@ func TestLinux_PlanStage(t *testing.T) { Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } + dockerTestMap := new(sync.Map) dockerTestMap.Store("foo", make(chan error, 1)) @@ -418,7 +428,7 @@ func TestLinux_PlanStage(t *testing.T) { WithPipeline(new(pipeline.Build)), WithRuntime(test.runtime), WithVelaClient(_client), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -480,6 +490,11 @@ func TestLinux_ExecStage(t *testing.T) { Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } + // setup tests tests := []struct { name string @@ -619,7 +634,7 @@ func TestLinux_ExecStage(t *testing.T) { WithVelaClient(_client), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -676,6 +691,11 @@ func TestLinux_DestroyStage(t *testing.T) { Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } + // setup tests tests := []struct { name string @@ -731,7 +751,7 @@ func TestLinux_DestroyStage(t *testing.T) { WithPipeline(new(pipeline.Build)), WithRuntime(test.runtime), WithVelaClient(_client), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) diff --git a/executor/local/build.go b/executor/local/build.go index 792418bf..46846ec4 100644 --- a/executor/local/build.go +++ b/executor/local/build.go @@ -25,9 +25,9 @@ func (c *client) CreateBuild(ctx context.Context) error { defer func() { build.Snapshot(c.build, nil, c.err, nil) }() // Check if storage client is initialized // and if storage is enabled - if c.Storage == nil { - return fmt.Errorf("storage client is not initialized") - } + //if c.Storage == nil && c.Storage. { + // return fmt.Errorf("storage client is not initialized") + //} // update the build fields c.build.SetStatus(constants.StatusRunning) @@ -297,7 +297,7 @@ func (c *client) ExecBuild(ctx context.Context) error { // check if the step should be skipped // // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Skip - skip, err := step.Skip(_step, c.build, c.build.GetStatus()) + skip, err := step.Skip(_step, c.build, c.build.GetStatus(), c.Storage) if err != nil { return fmt.Errorf("unable to plan step: %w", c.err) } diff --git a/executor/local/build_test.go b/executor/local/build_test.go index a20f7179..59ef0c8d 100644 --- a/executor/local/build_test.go +++ b/executor/local/build_test.go @@ -50,6 +50,11 @@ func TestLocal_CreateBuild(t *testing.T) { Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } + tests := []struct { name string failure bool @@ -91,7 +96,7 @@ func TestLocal_CreateBuild(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -148,6 +153,11 @@ func TestLocal_PlanBuild(t *testing.T) { Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } + tests := []struct { name string failure bool @@ -189,7 +199,7 @@ func TestLocal_PlanBuild(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -251,6 +261,10 @@ func TestLocal_AssembleBuild(t *testing.T) { Region: "", Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } streamRequests, done := message.MockStreamRequestsWithCancel(context.Background()) defer done() @@ -327,7 +341,7 @@ func TestLocal_AssembleBuild(t *testing.T) { WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -389,6 +403,10 @@ func TestLocal_ExecBuild(t *testing.T) { Region: "", Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } streamRequests, done := message.MockStreamRequestsWithCancel(context.Background()) defer done() @@ -450,7 +468,7 @@ func TestLocal_ExecBuild(t *testing.T) { WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -512,6 +530,10 @@ func TestLocal_StreamBuild(t *testing.T) { Region: "", Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } type planFuncType = func(context.Context, *pipeline.Container) error @@ -665,7 +687,7 @@ func TestLocal_StreamBuild(t *testing.T) { WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -747,6 +769,10 @@ func TestLocal_DestroyBuild(t *testing.T) { Region: "", Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } tests := []struct { name string @@ -804,7 +830,7 @@ func TestLocal_DestroyBuild(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) diff --git a/executor/local/opts.go b/executor/local/opts.go index ed4f1eee..3a8a856d 100644 --- a/executor/local/opts.go +++ b/executor/local/opts.go @@ -143,23 +143,17 @@ func withStreamRequests(s chan message.StreamRequest) Opt { } // WithStorage sets the storage in the executor client for Linux. -func WithStorage(s *storage.Setup) Opt { +func WithStorage(s *storage.Storage) Opt { return func(c *client) error { // check if the storage provided is empty if &s == nil { return fmt.Errorf("empty storage setup provided") } - // set the storage in the client - var err error - - c.Storage, err = storage.New(s) - if err != nil { - return fmt.Errorf("unable to create storage: %w", err) - } + c.Storage = *s if c.Storage == nil { - return fmt.Errorf("empty storage setup: %w", err) + return fmt.Errorf("empty storage setup: %v", s) } return nil diff --git a/executor/local/stage.go b/executor/local/stage.go index 05564596..a050a4f7 100644 --- a/executor/local/stage.go +++ b/executor/local/stage.go @@ -109,7 +109,7 @@ func (c *client) ExecStage(ctx context.Context, s *pipeline.Stage, m *sync.Map) // check if the step should be skipped // // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Skip - skip, err := step.Skip(_step, c.build, useStatus) + skip, err := step.Skip(_step, c.build, useStatus, c.Storage) if err != nil { return fmt.Errorf("unable to plan step: %w", c.err) } diff --git a/executor/local/stage_test.go b/executor/local/stage_test.go index b3f57dce..14955de6 100644 --- a/executor/local/stage_test.go +++ b/executor/local/stage_test.go @@ -62,6 +62,10 @@ func TestLocal_CreateStage(t *testing.T) { Region: "", Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } // setup tests tests := []struct { @@ -115,7 +119,7 @@ func TestLocal_CreateStage(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), - WithStorage(_storage), + WithStorage(&_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) diff --git a/executor/setup.go b/executor/setup.go index 148afaf0..8adf835d 100644 --- a/executor/setup.go +++ b/executor/setup.go @@ -62,7 +62,7 @@ type Setup struct { // id token request token for the build RequestToken string // storage client for interacting with storage resources - Storage *storage.Setup + Storage *storage.Storage } // Darwin creates and returns a Vela engine capable of @@ -78,10 +78,8 @@ func (s *Setup) Darwin() (Engine, error) { func (s *Setup) Linux() (Engine, error) { logrus.Trace("creating linux executor client from setup") - // create new Linux executor engine - // - // https://pkg.go.dev/github.com/go-vela/worker/executor/linux#New - return linux.New( + // create options for Linux executor + opts := []linux.Opt{ linux.WithBuild(s.Build), linux.WithMaxLogSize(s.MaxLogSize), linux.WithLogStreamingTimeout(s.LogStreamingTimeout), @@ -94,8 +92,18 @@ func (s *Setup) Linux() (Engine, error) { linux.WithVersion(s.Version), linux.WithLogger(s.Logger), linux.WithOutputCtn(s.OutputCtn), - linux.WithStorage(s.Storage), - ) + } + + // Conditionally add storage option + if s.Storage != nil { + opts = append(opts, linux.WithStorage(s.Storage)) + } + + // create new Linux executor engine + // + // https://pkg.go.dev/github.com/go-vela/worker/executor/linux#New + return linux.New(opts...) + } // Local creates and returns a Vela engine capable of @@ -103,10 +111,7 @@ func (s *Setup) Linux() (Engine, error) { func (s *Setup) Local() (Engine, error) { logrus.Trace("creating local executor client from setup") - // create new Local executor engine - // - // https://pkg.go.dev/github.com/go-vela/worker/executor/local#New - return local.New( + opts := []local.Opt{ local.WithBuild(s.Build), local.WithHostname(s.Hostname), local.WithPipeline(s.Pipeline), @@ -115,8 +120,17 @@ func (s *Setup) Local() (Engine, error) { local.WithVersion(s.Version), local.WithMockStdout(s.Mock), local.WithOutputCtn(s.OutputCtn), - local.WithStorage(s.Storage), - ) + } + + // Conditionally add storage option + if s.Storage != nil { + opts = append(opts, local.WithStorage(s.Storage)) + } + + // create new Local executor engine + // + // https://pkg.go.dev/github.com/go-vela/worker/executor/local#New + return local.New(opts...) } // Windows creates and returns a Vela engine capable of @@ -174,9 +188,9 @@ func (s *Setup) Validate() error { return fmt.Errorf("no Vela user provided in setup") } - // check if the storage client is provided + // If storage is provided, ensure it's enabled if s.Storage == nil { - return fmt.Errorf("no storage client provided in setup") + return fmt.Errorf("storage client provided but not enabled in setup") } // setup is valid diff --git a/executor/setup_test.go b/executor/setup_test.go index 2ecb6c96..62623251 100644 --- a/executor/setup_test.go +++ b/executor/setup_test.go @@ -81,6 +81,11 @@ func TestExecutor_Setup_Linux(t *testing.T) { Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } + want, err := linux.New( linux.WithBuild(_build), linux.WithMaxLogSize(2097152), @@ -90,7 +95,7 @@ func TestExecutor_Setup_Linux(t *testing.T) { linux.WithRuntime(_runtime), linux.WithVelaClient(_client), linux.WithVersion("v1.0.0"), - linux.WithStorage(_storage), + linux.WithStorage(&_s), ) if err != nil { t.Errorf("unable to create linux engine: %v", err) @@ -105,7 +110,7 @@ func TestExecutor_Setup_Linux(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: _storage, + Storage: &_s, } // run test @@ -148,6 +153,11 @@ func TestExecutor_Setup_Local(t *testing.T) { Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } + want, err := local.New( local.WithBuild(_build), local.WithHostname("localhost"), @@ -155,7 +165,7 @@ func TestExecutor_Setup_Local(t *testing.T) { local.WithRuntime(_runtime), local.WithVelaClient(_client), local.WithVersion("v1.0.0"), - local.WithStorage(_storage), + local.WithStorage(&_s), ) if err != nil { t.Errorf("unable to create local engine: %v", err) @@ -169,7 +179,7 @@ func TestExecutor_Setup_Local(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: _storage, + Storage: &_s, } // run test @@ -246,6 +256,11 @@ func TestExecutor_Setup_Validate(t *testing.T) { Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } + _emptyOwnerBuild := new(api.Build) _emptyOwnerBuild.SetRepo(new(api.Repo)) @@ -266,7 +281,7 @@ func TestExecutor_Setup_Validate(t *testing.T) { MaxLogSize: 2097152, Pipeline: _pipeline, Runtime: _runtime, - Storage: _storage, + Storage: &_s, }, failure: false, }, diff --git a/go.mod b/go.mod index 5e376e8c..050287fa 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,11 @@ module github.com/go-vela/worker go 1.24.6 +replace ( + github.com/go-vela/sdk-go => ../sdk-go + github.com/go-vela/server => ../server +) + require ( github.com/Masterminds/semver/v3 v3.4.0 github.com/distribution/reference v0.6.0 diff --git a/go.sum b/go.sum index 9175122e..a1de179b 100644 --- a/go.sum +++ b/go.sum @@ -125,10 +125,6 @@ github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/go-vela/sdk-go v0.27.2-0.20250929210753-8aa86d4e80fa h1:WicAh+R8C1/9WguXVr9wShMRiyTcrg12LOo+qMod/Sg= -github.com/go-vela/sdk-go v0.27.2-0.20250929210753-8aa86d4e80fa/go.mod h1:NujXkANYx1vkTfWeJXzk1lNJ4tzjnZF+0Pa1hqBrSTQ= -github.com/go-vela/server v0.27.3-0.20250929162825-a9dc2bb27fbd h1:YAbUQDQy1Z0OkpaxeaLd1vQ+yttqNP0G8fMiIr6SIbE= -github.com/go-vela/server v0.27.3-0.20250929162825-a9dc2bb27fbd/go.mod h1:ziaGq1cWl+y6C+Vz+FxvuiqZnDtzRneqDqTuzTra8VE= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= diff --git a/internal/step/skip.go b/internal/step/skip.go index b6f2ded7..033446d3 100644 --- a/internal/step/skip.go +++ b/internal/step/skip.go @@ -3,22 +3,27 @@ package step import ( + "fmt" "strings" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" + "github.com/go-vela/server/storage" ) // Skip creates the ruledata from the build and repository // information and returns true if the data does not match // the ruleset for the given container. -func Skip(c *pipeline.Container, b *api.Build, status string) (bool, error) { +func Skip(c *pipeline.Container, b *api.Build, status string, storage storage.Storage) (bool, error) { // check if the container provided is empty if c == nil { return true, nil } - + if !storage.StorageEnable() { + fmt.Printf("Skipping step %s, storage is nil", c.Name) + return true, nil + } event := b.GetEvent() action := b.GetEventAction() diff --git a/internal/step/skip_test.go b/internal/step/skip_test.go index 696f5df6..83277bae 100644 --- a/internal/step/skip_test.go +++ b/internal/step/skip_test.go @@ -8,6 +8,7 @@ import ( "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/storage" ) func TestStep_Skip(t *testing.T) { @@ -210,54 +211,76 @@ func TestStep_Skip(t *testing.T) { Name: "init", Number: 1, Pull: "always", + TestReport: pipeline.TestReport{ + Results: []string{"foo.xml", "bar.xml"}, + Attachments: []string{"foo.txt", "bar.txt"}, + }, + } + + s := &storage.Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://localhost:9000", + AccessKey: "minioadmin", + SecretKey: "minioadmin", + Bucket: "vela", } + _storage, _ := storage.New(s) tests := []struct { name string build *api.Build container *pipeline.Container + storage *storage.Storage want bool }{ { name: "build", build: _build, container: _container, + storage: &_storage, want: false, }, { name: "comment", build: _comment, container: _container, + storage: &_storage, want: false, }, { name: "deploy", build: _deploy, container: _container, + storage: &_storage, want: false, }, { name: "deployFromTag", build: _deployFromTag, container: _container, + storage: &_storage, want: false, }, { name: "schedule", build: _schedule, container: _container, + storage: &_storage, want: false, }, { name: "tag", build: _tag, container: _container, + storage: &_storage, want: false, }, { name: "skip nil", build: nil, container: nil, + storage: nil, want: true, }, } @@ -265,7 +288,7 @@ func TestStep_Skip(t *testing.T) { // run test for _, test := range tests { t.Run(test.name, func(t *testing.T) { - got, err := Skip(test.container, test.build, test.build.GetStatus()) + got, err := Skip(test.container, test.build, test.build.GetStatus(), *test.storage) if err != nil { t.Errorf("Skip returned error: %s", err) } diff --git a/router/middleware/executor/executor_test.go b/router/middleware/executor/executor_test.go index 11f69269..041619c1 100644 --- a/router/middleware/executor/executor_test.go +++ b/router/middleware/executor/executor_test.go @@ -44,6 +44,10 @@ func TestExecutor_Retrieve(t *testing.T) { Region: "", Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } want, err := executor.New(&executor.Setup{ Driver: constants.DriverLinux, @@ -52,7 +56,7 @@ func TestExecutor_Retrieve(t *testing.T) { Runtime: _runtime, Build: _build, Pipeline: new(pipeline.Build), - Storage: _storage, + Storage: &_s, }) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -95,6 +99,10 @@ func TestExecutor_Establish(t *testing.T) { Region: "", Secure: false, } + _s, err := storage.New(_storage) + if err != nil { + t.Errorf("unable to create storage engine: %v", err) + } want, err := executor.New(&executor.Setup{ Driver: constants.DriverLinux, @@ -103,7 +111,7 @@ func TestExecutor_Establish(t *testing.T) { Runtime: _runtime, Build: _build, Pipeline: new(pipeline.Build), - Storage: _storage, + Storage: &_s, }) if err != nil { t.Errorf("unable to create executor engine: %v", err) From 2b63a1c735cf5be6b195814a5f97c1c4b30589c1 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Tue, 14 Oct 2025 16:05:30 -0500 Subject: [PATCH 26/63] non working code --- cmd/vela-worker/exec.go | 80 +++++++++++++++----------------------- cmd/vela-worker/operate.go | 27 ++++++++++--- cmd/vela-worker/worker.go | 2 +- 3 files changed, 54 insertions(+), 55 deletions(-) diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index c34c8e0d..492a5ecf 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -159,11 +159,15 @@ func (w *Worker) exec(index int, config *api.Worker) error { execOutputCtn := *w.Config.Executor.OutputCtn execOutputCtn.ID = fmt.Sprintf("outputs_%s", p.ID) - logrus.Debugf("storage config: %+v", w.Config.Storage.Enable) - // dereference configured storage config and set the storage config for the executor - if w.Config.Storage.Enable { - logrus.Debugf("executor storage is enabled") - execStorage = w.Config.Executor.Storage + //logrus.Debugf("storage config: %+v", w.Config.Storage.Enable) + //// dereference configured storage config and set the storage config for the executor + //if w.Config.Storage.Enable { + // logrus.Debugf("executor storage is enabled") + // execStorage = w.Config.Executor.Storage + //} + execStorage = w.Storage + if execStorage == nil { + logrus.Debugf("executor storage is nil, skipping storage setup") } // create logger with extra metadata @@ -239,52 +243,30 @@ func (w *Worker) exec(index int, config *api.Worker) error { if err != nil { return err } + // setup the executor + // + // https://pkg.go.dev/github.com/go-vela/worker/executor#New + setup := &executor.Setup{ + Logger: logger, + Mock: w.Config.Mock, + Driver: w.Config.Executor.Driver, + MaxLogSize: w.Config.Executor.MaxLogSize, + LogStreamingTimeout: w.Config.Executor.LogStreamingTimeout, + EnforceTrustedRepos: w.Config.Executor.EnforceTrustedRepos, + PrivilegedImages: w.Config.Runtime.PrivilegedImages, + Client: execBuildClient, + Hostname: w.Config.API.Address.Hostname(), + Runtime: w.Runtime, + Build: item.Build, + Pipeline: p.Sanitize(w.Config.Runtime.Driver), + Version: v.Semantic(), + OutputCtn: &execOutputCtn, + } - if w.Config.Storage.Enable { - logrus.Debugf("executor storage is enabled") - logrus.Debugf("storage config: %+v", execStorage) - // setup the executor - // - // https://pkg.go.dev/github.com/go-vela/worker/executor#New - _executor, err = executor.New(&executor.Setup{ - Logger: logger, - Mock: w.Config.Mock, - Driver: w.Config.Executor.Driver, - MaxLogSize: w.Config.Executor.MaxLogSize, - LogStreamingTimeout: w.Config.Executor.LogStreamingTimeout, - EnforceTrustedRepos: w.Config.Executor.EnforceTrustedRepos, - PrivilegedImages: w.Config.Runtime.PrivilegedImages, - Client: execBuildClient, - Hostname: w.Config.API.Address.Hostname(), - Runtime: w.Runtime, - Build: item.Build, - Pipeline: p.Sanitize(w.Config.Runtime.Driver), - Version: v.Semantic(), - OutputCtn: &execOutputCtn, - Storage: execStorage, - }) - } else { - logrus.Debugf("executor storage is disabled") - // setup the executor - // - // https://pkg.go.dev/github.com/go-vela/worker/executor#New - _executor, err = executor.New(&executor.Setup{ - Logger: logger, - Mock: w.Config.Mock, - Driver: w.Config.Executor.Driver, - MaxLogSize: w.Config.Executor.MaxLogSize, - LogStreamingTimeout: w.Config.Executor.LogStreamingTimeout, - EnforceTrustedRepos: w.Config.Executor.EnforceTrustedRepos, - PrivilegedImages: w.Config.Runtime.PrivilegedImages, - Client: execBuildClient, - Hostname: w.Config.API.Address.Hostname(), - Runtime: w.Runtime, - Build: item.Build, - Pipeline: p.Sanitize(w.Config.Runtime.Driver), - Version: v.Semantic(), - OutputCtn: &execOutputCtn, - }) + if execStorage != nil { + setup.Storage = execStorage } + _executor, err = executor.New(setup) // add the executor to the worker w.Executors[index] = _executor diff --git a/cmd/vela-worker/operate.go b/cmd/vela-worker/operate.go index c819dfe9..6a8a1035 100644 --- a/cmd/vela-worker/operate.go +++ b/cmd/vela-worker/operate.go @@ -6,6 +6,7 @@ import ( "context" "time" + "github.com/go-vela/server/storage" "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" @@ -105,14 +106,30 @@ func (w *Worker) operate(ctx context.Context) error { // set bucket name in storage config w.Config.Storage.Bucket = stCreds.GetStorageBucket() logrus.Trace("bucket name: ", w.Config.Storage.Bucket) + + s, err := storage.New(w.Config.Storage) + if err != nil { + logrus.Error("storage setup failed") + // set to error as storage setup fails + w.updateWorkerStatus(registryWorker, constants.WorkerStatusError) + return err + } + w.Storage = &s + logrus.WithFields(logrus.Fields{ + "driver": w.Config.Storage.Driver, + "bucket": w.Config.Storage.Bucket, + "endpoint": w.Config.Storage.Endpoint, + }).Debug("storage initialized") } else { logrus.Trace("storage not enabled") // storage disabled; nothing to validate - w.Config.Storage.Driver = "" - w.Config.Storage.Endpoint = "" - w.Config.Storage.AccessKey = "" - w.Config.Storage.SecretKey = "" - w.Config.Storage.Bucket = "" + w.Storage = nil + logrus.Debug("storage disabled: worker storage unset") + //w.Config.Storage.Driver = "" + //w.Config.Storage.Endpoint = "" + //w.Config.Storage.AccessKey = "" + //w.Config.Storage.SecretKey = "" + //w.Config.Storage.Bucket = "" } // spawn goroutine for phoning home diff --git a/cmd/vela-worker/worker.go b/cmd/vela-worker/worker.go index 722883aa..6e8516b0 100644 --- a/cmd/vela-worker/worker.go +++ b/cmd/vela-worker/worker.go @@ -74,6 +74,6 @@ type ( RunningBuilds []*api.Build QueueCheckedIn bool RunningBuildsMutex sync.Mutex - Storage storage.Storage + Storage *storage.Storage } ) From 4bfe330a96caee9d5ceeea5aa50f7694ea2c5114 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Wed, 15 Oct 2025 11:10:54 -0500 Subject: [PATCH 27/63] working but skip step --- cmd/vela-worker/exec.go | 2 +- cmd/vela-worker/operate.go | 9 ++++++--- cmd/vela-worker/worker.go | 2 +- executor/executor_test.go | 14 +++++++------- executor/linux/build_test.go | 12 ++++++------ executor/linux/opts.go | 4 ++-- executor/linux/stage_test.go | 8 ++++---- executor/local/build_test.go | 12 ++++++------ executor/local/opts.go | 4 ++-- executor/local/stage_test.go | 2 +- executor/setup.go | 2 +- executor/setup_test.go | 10 +++++----- internal/step/skip.go | 12 +++++++++--- router/middleware/executor/executor_test.go | 5 +++-- 14 files changed, 54 insertions(+), 44 deletions(-) diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index 492a5ecf..96e501ee 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -29,7 +29,7 @@ import ( //nolint:funlen // ignore returning nil - don't want to crash worker func (w *Worker) exec(index int, config *api.Worker) error { var err error - var execStorage *storage.Storage + var execStorage storage.Storage var _executor executor.Engine // setup the version diff --git a/cmd/vela-worker/operate.go b/cmd/vela-worker/operate.go index 6a8a1035..4d69aa10 100644 --- a/cmd/vela-worker/operate.go +++ b/cmd/vela-worker/operate.go @@ -85,11 +85,11 @@ func (w *Worker) operate(ctx context.Context) error { logrus.Tracef("error getting storage creds: %v", err) return err } - w.Config.Storage.Enable = stCreds.GetEnabled() + //w.Config.Storage.Enable = stCreds.GetEnabled() logrus.Trace("Storage enabled: ", w.Config.Storage.Enable) - if w.Config.Storage.Enable { + if stCreds.GetEnabled() { logrus.Trace("storage enabled") // if an address was given at start up, use that — else use what is returned from server if len(w.Config.Storage.Endpoint) == 0 { @@ -114,16 +114,19 @@ func (w *Worker) operate(ctx context.Context) error { w.updateWorkerStatus(registryWorker, constants.WorkerStatusError) return err } - w.Storage = &s + w.Storage = s logrus.WithFields(logrus.Fields{ "driver": w.Config.Storage.Driver, "bucket": w.Config.Storage.Bucket, "endpoint": w.Config.Storage.Endpoint, }).Debug("storage initialized") + w.Config.Storage.Enable = true + } else { logrus.Trace("storage not enabled") // storage disabled; nothing to validate w.Storage = nil + w.Config.Storage.Enable = false logrus.Debug("storage disabled: worker storage unset") //w.Config.Storage.Driver = "" //w.Config.Storage.Endpoint = "" diff --git a/cmd/vela-worker/worker.go b/cmd/vela-worker/worker.go index 6e8516b0..722883aa 100644 --- a/cmd/vela-worker/worker.go +++ b/cmd/vela-worker/worker.go @@ -74,6 +74,6 @@ type ( RunningBuilds []*api.Build QueueCheckedIn bool RunningBuildsMutex sync.Mutex - Storage *storage.Storage + Storage storage.Storage } ) diff --git a/executor/executor_test.go b/executor/executor_test.go index c295b927..f194a58f 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -76,7 +76,7 @@ func TestExecutor_New(t *testing.T) { linux.WithRuntime(_runtime), linux.WithVelaClient(_client), linux.WithVersion("v1.0.0"), - linux.WithStorage(&_sT), + linux.WithStorage(_sT), ) if err != nil { t.Errorf("unable to create linux engine: %v", err) @@ -112,7 +112,7 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: &_sF, + Storage: _sF, }, want: nil, equal: reflect.DeepEqual, @@ -128,7 +128,7 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: &_sT, + Storage: _sT, }, want: _linux, equal: linux.Equal, @@ -143,7 +143,7 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: &_sT, + Storage: _sT, }, want: _local, equal: local.Equal, @@ -158,7 +158,7 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: &_sT, + Storage: _sT, }, want: nil, equal: reflect.DeepEqual, @@ -173,7 +173,7 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: &_sT, + Storage: _sT, }, want: nil, equal: reflect.DeepEqual, @@ -188,7 +188,7 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: &_sT, + Storage: _sT, }, want: nil, equal: reflect.DeepEqual, diff --git a/executor/linux/build_test.go b/executor/linux/build_test.go index 49bbe2e2..ab75f60c 100644 --- a/executor/linux/build_test.go +++ b/executor/linux/build_test.go @@ -203,7 +203,7 @@ func TestLinux_CreateBuild(t *testing.T) { WithRuntime(_runtime), WithVelaClient(_client), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -406,7 +406,7 @@ func TestLinux_PlanBuild(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithVelaClient(_client), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -717,7 +717,7 @@ func TestLinux_AssembleBuild(t *testing.T) { WithVelaClient(_client), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -987,7 +987,7 @@ func TestLinux_ExecBuild(t *testing.T) { WithVelaClient(_client), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -1653,7 +1653,7 @@ func TestLinux_StreamBuild(t *testing.T) { WithLogStreamingTimeout(1*time.Second), WithVelaClient(_client), withStreamRequests(streamRequests), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -1956,7 +1956,7 @@ func TestLinux_DestroyBuild(t *testing.T) { WithRuntime(_runtime), WithVelaClient(_client), WithOutputCtn(testOutputsCtn()), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) diff --git a/executor/linux/opts.go b/executor/linux/opts.go index 0599e8b1..f2b1df62 100644 --- a/executor/linux/opts.go +++ b/executor/linux/opts.go @@ -212,7 +212,7 @@ func withStreamRequests(s chan message.StreamRequest) Opt { } // WithStorage sets the storage in the executor client for Linux. -func WithStorage(s *storage.Storage) Opt { +func WithStorage(s storage.Storage) Opt { return func(c *client) error { c.Logger.Trace("configuring storage in linux executor client") @@ -221,7 +221,7 @@ func WithStorage(s *storage.Storage) Opt { return fmt.Errorf("empty storage setup provided") } - c.Storage = *s + c.Storage = s if c.Storage == nil { return fmt.Errorf("empty storage setup: %v", s) diff --git a/executor/linux/stage_test.go b/executor/linux/stage_test.go index 7aed2171..fc6a9954 100644 --- a/executor/linux/stage_test.go +++ b/executor/linux/stage_test.go @@ -188,7 +188,7 @@ func TestLinux_CreateStage(t *testing.T) { WithPipeline(_pipeline), WithRuntime(test.runtime), WithVelaClient(_client), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -428,7 +428,7 @@ func TestLinux_PlanStage(t *testing.T) { WithPipeline(new(pipeline.Build)), WithRuntime(test.runtime), WithVelaClient(_client), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -634,7 +634,7 @@ func TestLinux_ExecStage(t *testing.T) { WithVelaClient(_client), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -751,7 +751,7 @@ func TestLinux_DestroyStage(t *testing.T) { WithPipeline(new(pipeline.Build)), WithRuntime(test.runtime), WithVelaClient(_client), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) diff --git a/executor/local/build_test.go b/executor/local/build_test.go index 59ef0c8d..c3852913 100644 --- a/executor/local/build_test.go +++ b/executor/local/build_test.go @@ -96,7 +96,7 @@ func TestLocal_CreateBuild(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -199,7 +199,7 @@ func TestLocal_PlanBuild(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -341,7 +341,7 @@ func TestLocal_AssembleBuild(t *testing.T) { WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -468,7 +468,7 @@ func TestLocal_ExecBuild(t *testing.T) { WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -687,7 +687,7 @@ func TestLocal_StreamBuild(t *testing.T) { WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -830,7 +830,7 @@ func TestLocal_DestroyBuild(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) diff --git a/executor/local/opts.go b/executor/local/opts.go index 3a8a856d..0def904d 100644 --- a/executor/local/opts.go +++ b/executor/local/opts.go @@ -143,14 +143,14 @@ func withStreamRequests(s chan message.StreamRequest) Opt { } // WithStorage sets the storage in the executor client for Linux. -func WithStorage(s *storage.Storage) Opt { +func WithStorage(s storage.Storage) Opt { return func(c *client) error { // check if the storage provided is empty if &s == nil { return fmt.Errorf("empty storage setup provided") } - c.Storage = *s + c.Storage = s if c.Storage == nil { return fmt.Errorf("empty storage setup: %v", s) diff --git a/executor/local/stage_test.go b/executor/local/stage_test.go index 14955de6..416b9908 100644 --- a/executor/local/stage_test.go +++ b/executor/local/stage_test.go @@ -119,7 +119,7 @@ func TestLocal_CreateStage(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), - WithStorage(&_s), + WithStorage(_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) diff --git a/executor/setup.go b/executor/setup.go index 8adf835d..dde97cc7 100644 --- a/executor/setup.go +++ b/executor/setup.go @@ -62,7 +62,7 @@ type Setup struct { // id token request token for the build RequestToken string // storage client for interacting with storage resources - Storage *storage.Storage + Storage storage.Storage } // Darwin creates and returns a Vela engine capable of diff --git a/executor/setup_test.go b/executor/setup_test.go index 62623251..87999425 100644 --- a/executor/setup_test.go +++ b/executor/setup_test.go @@ -95,7 +95,7 @@ func TestExecutor_Setup_Linux(t *testing.T) { linux.WithRuntime(_runtime), linux.WithVelaClient(_client), linux.WithVersion("v1.0.0"), - linux.WithStorage(&_s), + linux.WithStorage(_s), ) if err != nil { t.Errorf("unable to create linux engine: %v", err) @@ -110,7 +110,7 @@ func TestExecutor_Setup_Linux(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: &_s, + Storage: _s, } // run test @@ -165,7 +165,7 @@ func TestExecutor_Setup_Local(t *testing.T) { local.WithRuntime(_runtime), local.WithVelaClient(_client), local.WithVersion("v1.0.0"), - local.WithStorage(&_s), + local.WithStorage(_s), ) if err != nil { t.Errorf("unable to create local engine: %v", err) @@ -179,7 +179,7 @@ func TestExecutor_Setup_Local(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: &_s, + Storage: _s, } // run test @@ -281,7 +281,7 @@ func TestExecutor_Setup_Validate(t *testing.T) { MaxLogSize: 2097152, Pipeline: _pipeline, Runtime: _runtime, - Storage: &_s, + Storage: _s, }, failure: false, }, diff --git a/internal/step/skip.go b/internal/step/skip.go index 033446d3..a2173055 100644 --- a/internal/step/skip.go +++ b/internal/step/skip.go @@ -16,14 +16,20 @@ import ( // information and returns true if the data does not match // the ruleset for the given container. func Skip(c *pipeline.Container, b *api.Build, status string, storage storage.Storage) (bool, error) { + fmt.Printf("evaluating step %s for skip\n", c.Name) // check if the container provided is empty if c == nil { return true, nil } - if !storage.StorageEnable() { - fmt.Printf("Skipping step %s, storage is nil", c.Name) - return true, nil + + if !c.TestReport.Empty() { + fmt.Printf("test report step detected\n") + if !storage.StorageEnable() { + fmt.Printf("Skipping step %s, storage is nil", c.Name) + return true, nil + } } + event := b.GetEvent() action := b.GetEventAction() diff --git a/router/middleware/executor/executor_test.go b/router/middleware/executor/executor_test.go index 041619c1..fc6676fd 100644 --- a/router/middleware/executor/executor_test.go +++ b/router/middleware/executor/executor_test.go @@ -56,7 +56,7 @@ func TestExecutor_Retrieve(t *testing.T) { Runtime: _runtime, Build: _build, Pipeline: new(pipeline.Build), - Storage: &_s, + Storage: _s, }) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -99,6 +99,7 @@ func TestExecutor_Establish(t *testing.T) { Region: "", Secure: false, } + _s, err := storage.New(_storage) if err != nil { t.Errorf("unable to create storage engine: %v", err) @@ -111,7 +112,7 @@ func TestExecutor_Establish(t *testing.T) { Runtime: _runtime, Build: _build, Pipeline: new(pipeline.Build), - Storage: &_s, + Storage: _s, }) if err != nil { t.Errorf("unable to create executor engine: %v", err) From 0fe916861182c8f2b5d0eb521d567f25def167b6 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Wed, 15 Oct 2025 14:19:49 -0500 Subject: [PATCH 28/63] working with enabled --- cmd/vela-worker/exec.go | 1 + cmd/vela-worker/operate.go | 4 +++- executor/setup.go | 1 + internal/step/skip.go | 7 ++++--- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index 96e501ee..fc37b950 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -264,6 +264,7 @@ func (w *Worker) exec(index int, config *api.Worker) error { } if execStorage != nil { + fmt.Printf("setting up executor storage\n") setup.Storage = execStorage } _executor, err = executor.New(setup) diff --git a/cmd/vela-worker/operate.go b/cmd/vela-worker/operate.go index 4d69aa10..5bf82681 100644 --- a/cmd/vela-worker/operate.go +++ b/cmd/vela-worker/operate.go @@ -107,6 +107,9 @@ func (w *Worker) operate(ctx context.Context) error { w.Config.Storage.Bucket = stCreds.GetStorageBucket() logrus.Trace("bucket name: ", w.Config.Storage.Bucket) + // set storage enabled to true + w.Config.Storage.Enable = stCreds.GetEnabled() + s, err := storage.New(w.Config.Storage) if err != nil { logrus.Error("storage setup failed") @@ -120,7 +123,6 @@ func (w *Worker) operate(ctx context.Context) error { "bucket": w.Config.Storage.Bucket, "endpoint": w.Config.Storage.Endpoint, }).Debug("storage initialized") - w.Config.Storage.Enable = true } else { logrus.Trace("storage not enabled") diff --git a/executor/setup.go b/executor/setup.go index dde97cc7..1058e74e 100644 --- a/executor/setup.go +++ b/executor/setup.go @@ -96,6 +96,7 @@ func (s *Setup) Linux() (Engine, error) { // Conditionally add storage option if s.Storage != nil { + fmt.Printf("Adding storage to linux executor\n") opts = append(opts, linux.WithStorage(s.Storage)) } diff --git a/internal/step/skip.go b/internal/step/skip.go index a2173055..b96e8db9 100644 --- a/internal/step/skip.go +++ b/internal/step/skip.go @@ -23,13 +23,14 @@ func Skip(c *pipeline.Container, b *api.Build, status string, storage storage.St } if !c.TestReport.Empty() { - fmt.Printf("test report step detected\n") + if storage == nil { + return true, nil + } if !storage.StorageEnable() { - fmt.Printf("Skipping step %s, storage is nil", c.Name) return true, nil } + return false, nil } - event := b.GetEvent() action := b.GetEventAction() From 419a68767ad8f17a99141eead026ae922a600427 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Thu, 16 Oct 2025 10:27:46 -0500 Subject: [PATCH 29/63] working with enabled --- cmd/vela-worker/exec.go | 19 +++++++++---------- cmd/vela-worker/operate.go | 15 ++++----------- executor/setup.go | 2 +- 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index fc37b950..87f209cc 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -159,17 +159,13 @@ func (w *Worker) exec(index int, config *api.Worker) error { execOutputCtn := *w.Config.Executor.OutputCtn execOutputCtn.ID = fmt.Sprintf("outputs_%s", p.ID) - //logrus.Debugf("storage config: %+v", w.Config.Storage.Enable) - //// dereference configured storage config and set the storage config for the executor - //if w.Config.Storage.Enable { - // logrus.Debugf("executor storage is enabled") - // execStorage = w.Config.Executor.Storage - //} - execStorage = w.Storage - if execStorage == nil { + if w.Storage != nil { + execStorage = w.Storage + logrus.Debugf("executor storage is available, setting up storage") + } else { logrus.Debugf("executor storage is nil, skipping storage setup") } - + // create logger with extra metadata // // https://pkg.go.dev/github.com/sirupsen/logrus#WithFields @@ -268,7 +264,10 @@ func (w *Worker) exec(index int, config *api.Worker) error { setup.Storage = execStorage } _executor, err = executor.New(setup) - + if err != nil { + logger.Errorf("unable to setup executor: %v", err) + return err + } // add the executor to the worker w.Executors[index] = _executor diff --git a/cmd/vela-worker/operate.go b/cmd/vela-worker/operate.go index 5bf82681..8697d61c 100644 --- a/cmd/vela-worker/operate.go +++ b/cmd/vela-worker/operate.go @@ -77,6 +77,7 @@ func (w *Worker) operate(ctx context.Context) error { // set to error as queue setup fails w.updateWorkerStatus(registryWorker, constants.WorkerStatusError) } + // getting storage creds logrus.Trace("getting storage s3 creds..") // fetching queue credentials using registration token @@ -85,7 +86,6 @@ func (w *Worker) operate(ctx context.Context) error { logrus.Tracef("error getting storage creds: %v", err) return err } - //w.Config.Storage.Enable = stCreds.GetEnabled() logrus.Trace("Storage enabled: ", w.Config.Storage.Enable) @@ -94,22 +94,20 @@ func (w *Worker) operate(ctx context.Context) error { // if an address was given at start up, use that — else use what is returned from server if len(w.Config.Storage.Endpoint) == 0 { w.Config.Storage.Endpoint = stCreds.GetStorageAddress() - logrus.Trace("storage address: ", w.Config.Storage.Driver) } // set access key in storage config w.Config.Storage.AccessKey = stCreds.GetAccessKey() - logrus.Trace("access key: ", w.Config.Storage.AccessKey) + // set secret key in storage config w.Config.Storage.SecretKey = stCreds.GetSecretKey() // set bucket name in storage config w.Config.Storage.Bucket = stCreds.GetStorageBucket() - logrus.Trace("bucket name: ", w.Config.Storage.Bucket) // set storage enabled to true w.Config.Storage.Enable = stCreds.GetEnabled() - + s, err := storage.New(w.Config.Storage) if err != nil { logrus.Error("storage setup failed") @@ -127,14 +125,9 @@ func (w *Worker) operate(ctx context.Context) error { } else { logrus.Trace("storage not enabled") // storage disabled; nothing to validate - w.Storage = nil w.Config.Storage.Enable = false + w.Storage = nil logrus.Debug("storage disabled: worker storage unset") - //w.Config.Storage.Driver = "" - //w.Config.Storage.Endpoint = "" - //w.Config.Storage.AccessKey = "" - //w.Config.Storage.SecretKey = "" - //w.Config.Storage.Bucket = "" } // spawn goroutine for phoning home diff --git a/executor/setup.go b/executor/setup.go index 1058e74e..3c3f196d 100644 --- a/executor/setup.go +++ b/executor/setup.go @@ -190,7 +190,7 @@ func (s *Setup) Validate() error { } // If storage is provided, ensure it's enabled - if s.Storage == nil { + if s.Storage != nil && !s.Storage.StorageEnable() { return fmt.Errorf("storage client provided but not enabled in setup") } From 605d9a4e176490959a0b8a231f000b76c77b3c81 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Thu, 23 Oct 2025 11:56:43 -0500 Subject: [PATCH 30/63] fix skip test --- cmd/vela-worker/operate.go | 2 -- executor/executor_test.go | 1 + internal/step/skip.go | 4 +--- internal/step/skip_test.go | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/cmd/vela-worker/operate.go b/cmd/vela-worker/operate.go index 8697d61c..fec2c1e8 100644 --- a/cmd/vela-worker/operate.go +++ b/cmd/vela-worker/operate.go @@ -87,8 +87,6 @@ func (w *Worker) operate(ctx context.Context) error { return err } - logrus.Trace("Storage enabled: ", w.Config.Storage.Enable) - if stCreds.GetEnabled() { logrus.Trace("storage enabled") // if an address was given at start up, use that — else use what is returned from server diff --git a/executor/executor_test.go b/executor/executor_test.go index f194a58f..b7ecb6d1 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -89,6 +89,7 @@ func TestExecutor_New(t *testing.T) { local.WithRuntime(_runtime), local.WithVelaClient(_client), local.WithVersion("v1.0.0"), + local.WithStorage(_sT), ) if err != nil { t.Errorf("unable to create local engine: %v", err) diff --git a/internal/step/skip.go b/internal/step/skip.go index b96e8db9..2f378d51 100644 --- a/internal/step/skip.go +++ b/internal/step/skip.go @@ -3,7 +3,6 @@ package step import ( - "fmt" "strings" api "github.com/go-vela/server/api/types" @@ -16,14 +15,13 @@ import ( // information and returns true if the data does not match // the ruleset for the given container. func Skip(c *pipeline.Container, b *api.Build, status string, storage storage.Storage) (bool, error) { - fmt.Printf("evaluating step %s for skip\n", c.Name) // check if the container provided is empty if c == nil { return true, nil } if !c.TestReport.Empty() { - if storage == nil { + if &storage == nil { return true, nil } if !storage.StorageEnable() { diff --git a/internal/step/skip_test.go b/internal/step/skip_test.go index 83277bae..8c9a906e 100644 --- a/internal/step/skip_test.go +++ b/internal/step/skip_test.go @@ -280,7 +280,7 @@ func TestStep_Skip(t *testing.T) { name: "skip nil", build: nil, container: nil, - storage: nil, + storage: &_storage, want: true, }, } From 76efee5401824b0fd58e4be9f935defecbfa683b Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Thu, 23 Oct 2025 13:24:41 -0500 Subject: [PATCH 31/63] point to server branch --- go.mod | 9 ++------- go.sum | 6 ++++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 050287fa..f059ad2d 100644 --- a/go.mod +++ b/go.mod @@ -2,19 +2,14 @@ module github.com/go-vela/worker go 1.24.6 -replace ( - github.com/go-vela/sdk-go => ../sdk-go - github.com/go-vela/server => ../server -) - require ( github.com/Masterminds/semver/v3 v3.4.0 github.com/distribution/reference v0.6.0 github.com/docker/docker v28.3.3+incompatible github.com/docker/go-units v0.5.0 github.com/gin-gonic/gin v1.10.1 - github.com/go-vela/sdk-go v0.27.2-0.20250929210753-8aa86d4e80fa - github.com/go-vela/server v0.27.3-0.20250929162825-a9dc2bb27fbd + github.com/go-vela/sdk-go v0.27.2-0.20251023182037-f8a9d8e9a0a9 + github.com/go-vela/server v0.27.3-0.20251023181434-3790af6cb5db github.com/golang-jwt/jwt/v5 v5.3.0 github.com/google/go-cmp v0.7.0 github.com/joho/godotenv v1.5.1 diff --git a/go.sum b/go.sum index a1de179b..0dec4754 100644 --- a/go.sum +++ b/go.sum @@ -125,6 +125,12 @@ github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-vela/sdk-go v0.27.2-0.20250929210753-8aa86d4e80fa/go.mod h1:NujXkANYx1vkTfWeJXzk1lNJ4tzjnZF+0Pa1hqBrSTQ= +github.com/go-vela/sdk-go v0.27.2-0.20251023182037-f8a9d8e9a0a9 h1:jHgyONH8Mt91qkDtbLyN+F4RFoZSZboYZY3U5FoB8vw= +github.com/go-vela/sdk-go v0.27.2-0.20251023182037-f8a9d8e9a0a9/go.mod h1:wJF3oFa934D+7O+/7ruDz3h51FeePFZPP40zt1kg7Ac= +github.com/go-vela/server v0.27.3-0.20250929162825-a9dc2bb27fbd/go.mod h1:ziaGq1cWl+y6C+Vz+FxvuiqZnDtzRneqDqTuzTra8VE= +github.com/go-vela/server v0.27.3-0.20251023181434-3790af6cb5db h1:4/7TFAENWqGloUg2TAJNDKEj4b/t/f2iLKOLJuMUsXU= +github.com/go-vela/server v0.27.3-0.20251023181434-3790af6cb5db/go.mod h1:ziaGq1cWl+y6C+Vz+FxvuiqZnDtzRneqDqTuzTra8VE= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= From 3991c623e00bbab01c2589bb52e4bd2d0cac06c2 Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Wed, 12 Nov 2025 15:08:41 -0600 Subject: [PATCH 32/63] update for presigned url --- cmd/vela-worker/exec.go | 9 +++++++-- cmd/vela-worker/operate.go | 6 ++++-- cmd/vela-worker/run.go | 2 +- executor/executor_test.go | 2 ++ executor/linux/build_test.go | 1 + executor/linux/opts.go | 4 ++-- executor/linux/outputs.go | 11 +++++++++-- executor/linux/testattachments.go | 4 ++-- executor/local/build_test.go | 4 ++++ executor/local/opts.go | 3 ++- executor/local/stage_test.go | 1 + executor/setup.go | 2 +- internal/step/skip.go | 5 ++++- router/middleware/executor/executor_test.go | 1 + 14 files changed, 41 insertions(+), 14 deletions(-) diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index 87f209cc..e87f31bc 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -10,7 +10,6 @@ import ( "sync" "time" - "github.com/go-vela/server/storage" "github.com/sirupsen/logrus" "github.com/go-vela/sdk-go/vela" @@ -18,6 +17,7 @@ import ( "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" "github.com/go-vela/server/queue/models" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor" "github.com/go-vela/worker/runtime" "github.com/go-vela/worker/version" @@ -29,7 +29,9 @@ import ( //nolint:funlen // ignore returning nil - don't want to crash worker func (w *Worker) exec(index int, config *api.Worker) error { var err error + var execStorage storage.Storage + var _executor executor.Engine // setup the version @@ -161,11 +163,12 @@ func (w *Worker) exec(index int, config *api.Worker) error { if w.Storage != nil { execStorage = w.Storage + logrus.Debugf("executor storage is available, setting up storage") } else { logrus.Debugf("executor storage is nil, skipping storage setup") } - + // create logger with extra metadata // // https://pkg.go.dev/github.com/sirupsen/logrus#WithFields @@ -261,8 +264,10 @@ func (w *Worker) exec(index int, config *api.Worker) error { if execStorage != nil { fmt.Printf("setting up executor storage\n") + setup.Storage = execStorage } + _executor, err = executor.New(setup) if err != nil { logger.Errorf("unable to setup executor: %v", err) diff --git a/cmd/vela-worker/operate.go b/cmd/vela-worker/operate.go index fec2c1e8..35fdc207 100644 --- a/cmd/vela-worker/operate.go +++ b/cmd/vela-worker/operate.go @@ -6,13 +6,13 @@ import ( "context" "time" - "github.com/go-vela/server/storage" "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/constants" "github.com/go-vela/server/queue" + "github.com/go-vela/server/storage" ) // operate is a helper function to initiate all @@ -111,20 +111,22 @@ func (w *Worker) operate(ctx context.Context) error { logrus.Error("storage setup failed") // set to error as storage setup fails w.updateWorkerStatus(registryWorker, constants.WorkerStatusError) + return err } + w.Storage = s logrus.WithFields(logrus.Fields{ "driver": w.Config.Storage.Driver, "bucket": w.Config.Storage.Bucket, "endpoint": w.Config.Storage.Endpoint, }).Debug("storage initialized") - } else { logrus.Trace("storage not enabled") // storage disabled; nothing to validate w.Config.Storage.Enable = false w.Storage = nil + logrus.Debug("storage disabled: worker storage unset") } diff --git a/cmd/vela-worker/run.go b/cmd/vela-worker/run.go index bfda637f..bd5ffb8c 100644 --- a/cmd/vela-worker/run.go +++ b/cmd/vela-worker/run.go @@ -8,7 +8,6 @@ import ( "net/url" "github.com/gin-gonic/gin" - "github.com/go-vela/server/storage" "github.com/sirupsen/logrus" "github.com/urfave/cli/v3" @@ -18,6 +17,7 @@ import ( "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" "github.com/go-vela/server/queue" + "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor" "github.com/go-vela/worker/runtime" ) diff --git a/executor/executor_test.go b/executor/executor_test.go index b7ecb6d1..f888e6ae 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -48,6 +48,7 @@ func TestExecutor_New(t *testing.T) { Region: "", Secure: false, } + _sT, err := storage.New(_storageT) if err != nil { t.Errorf("unable to create storage engine: %v", err) @@ -63,6 +64,7 @@ func TestExecutor_New(t *testing.T) { Region: "", Secure: false, } + _sF, err := storage.New(_storageF) if err != nil { t.Errorf("unable to create storage engine: %v", err) diff --git a/executor/linux/build_test.go b/executor/linux/build_test.go index ab75f60c..49f2acf6 100644 --- a/executor/linux/build_test.go +++ b/executor/linux/build_test.go @@ -280,6 +280,7 @@ func TestLinux_PlanBuild(t *testing.T) { Region: "", Secure: false, } + _s, err := storage.New(_storage) if err != nil { t.Errorf("unable to create storage engine: %v", err) diff --git a/executor/linux/opts.go b/executor/linux/opts.go index f2b1df62..33d982c8 100644 --- a/executor/linux/opts.go +++ b/executor/linux/opts.go @@ -217,14 +217,14 @@ func WithStorage(s storage.Storage) Opt { c.Logger.Trace("configuring storage in linux executor client") // check if the storage provided is empty - if &s == nil { + if s == nil { return fmt.Errorf("empty storage setup provided") } c.Storage = s if c.Storage == nil { - return fmt.Errorf("empty storage setup: %v", s) + return fmt.Errorf("empty storage setup in linux executor client") } return nil diff --git a/executor/linux/outputs.go b/executor/linux/outputs.go index 9ba867e4..2b060206 100644 --- a/executor/linux/outputs.go +++ b/executor/linux/outputs.go @@ -258,10 +258,17 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, file return fmt.Errorf("unable to upload object %s: %w", fileName, err) } - logger.Debugf("successfully uploaded file %s (%d bytes)", fileName, size) + presignURL, err := o.client.Storage.PresignedGetObject(ctx, &api.Object{ + ObjectName: objectName, + Bucket: api.Bucket{BucketName: o.client.Storage.GetBucket(ctx)}, + FilePath: filePath, + }) + if err != nil { + return fmt.Errorf("unable to generate presign URL for %s: %w", fileName, err) + } // create test attachment record in database after successful upload - err = o.client.CreateTestAttachment(fileName, filePath, size, tr) + err = o.client.CreateTestAttachment(fileName, presignURL, size, tr) if err != nil { logger.Errorf("unable to create test attachment record for %s: %v", fileName, err) // don't return error here to avoid blocking the upload process diff --git a/executor/linux/testattachments.go b/executor/linux/testattachments.go index 026fd407..ef217957 100644 --- a/executor/linux/testattachments.go +++ b/executor/linux/testattachments.go @@ -13,7 +13,7 @@ import ( // CreateTestAttachment creates a test attachment record in the database // after a file has been successfully uploaded to storage. -func (c *client) CreateTestAttachment(fileName, filePath string, size int64, tr *api.TestReport) error { +func (c *client) CreateTestAttachment(fileName, presignURL string, size int64, tr *api.TestReport) error { // extract file extension and type information fileExt := filepath.Ext(fileName) @@ -34,7 +34,7 @@ func (c *client) CreateTestAttachment(fileName, filePath string, size int64, tr ObjectPath: &objectPath, FileSize: &size, FileType: &fileExt, - PresignedURL: nil, // will be generated by the API/storage service + PresignedURL: &presignURL, CreatedAt: &createdAt, } diff --git a/executor/local/build_test.go b/executor/local/build_test.go index c3852913..e85af048 100644 --- a/executor/local/build_test.go +++ b/executor/local/build_test.go @@ -261,6 +261,7 @@ func TestLocal_AssembleBuild(t *testing.T) { Region: "", Secure: false, } + _s, err := storage.New(_storage) if err != nil { t.Errorf("unable to create storage engine: %v", err) @@ -403,6 +404,7 @@ func TestLocal_ExecBuild(t *testing.T) { Region: "", Secure: false, } + _s, err := storage.New(_storage) if err != nil { t.Errorf("unable to create storage engine: %v", err) @@ -530,6 +532,7 @@ func TestLocal_StreamBuild(t *testing.T) { Region: "", Secure: false, } + _s, err := storage.New(_storage) if err != nil { t.Errorf("unable to create storage engine: %v", err) @@ -769,6 +772,7 @@ func TestLocal_DestroyBuild(t *testing.T) { Region: "", Secure: false, } + _s, err := storage.New(_storage) if err != nil { t.Errorf("unable to create storage engine: %v", err) diff --git a/executor/local/opts.go b/executor/local/opts.go index 0def904d..fb012db8 100644 --- a/executor/local/opts.go +++ b/executor/local/opts.go @@ -146,10 +146,11 @@ func withStreamRequests(s chan message.StreamRequest) Opt { func WithStorage(s storage.Storage) Opt { return func(c *client) error { // check if the storage provided is empty - if &s == nil { + if s == nil { return fmt.Errorf("empty storage setup provided") } + // set the storage in the client c.Storage = s if c.Storage == nil { diff --git a/executor/local/stage_test.go b/executor/local/stage_test.go index 416b9908..b0b93b7b 100644 --- a/executor/local/stage_test.go +++ b/executor/local/stage_test.go @@ -62,6 +62,7 @@ func TestLocal_CreateStage(t *testing.T) { Region: "", Secure: false, } + _s, err := storage.New(_storage) if err != nil { t.Errorf("unable to create storage engine: %v", err) diff --git a/executor/setup.go b/executor/setup.go index 3c3f196d..10f37f24 100644 --- a/executor/setup.go +++ b/executor/setup.go @@ -97,6 +97,7 @@ func (s *Setup) Linux() (Engine, error) { // Conditionally add storage option if s.Storage != nil { fmt.Printf("Adding storage to linux executor\n") + opts = append(opts, linux.WithStorage(s.Storage)) } @@ -104,7 +105,6 @@ func (s *Setup) Linux() (Engine, error) { // // https://pkg.go.dev/github.com/go-vela/worker/executor/linux#New return linux.New(opts...) - } // Local creates and returns a Vela engine capable of diff --git a/internal/step/skip.go b/internal/step/skip.go index 2f378d51..e1e05165 100644 --- a/internal/step/skip.go +++ b/internal/step/skip.go @@ -21,14 +21,17 @@ func Skip(c *pipeline.Container, b *api.Build, status string, storage storage.St } if !c.TestReport.Empty() { - if &storage == nil { + if storage == nil { return true, nil } + if !storage.StorageEnable() { return true, nil } + return false, nil } + event := b.GetEvent() action := b.GetEventAction() diff --git a/router/middleware/executor/executor_test.go b/router/middleware/executor/executor_test.go index fc6676fd..c94f032b 100644 --- a/router/middleware/executor/executor_test.go +++ b/router/middleware/executor/executor_test.go @@ -44,6 +44,7 @@ func TestExecutor_Retrieve(t *testing.T) { Region: "", Secure: false, } + _s, err := storage.New(_storage) if err != nil { t.Errorf("unable to create storage engine: %v", err) From 9a6b3147c8aa592fc9eaad8584061eb1030c9fec Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Mon, 15 Dec 2025 15:35:38 -0600 Subject: [PATCH 33/63] fix liniting --- runtime/kubernetes/container.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/kubernetes/container.go b/runtime/kubernetes/container.go index 79dc8731..65bae84e 100644 --- a/runtime/kubernetes/container.go +++ b/runtime/kubernetes/container.go @@ -79,7 +79,7 @@ func (c *client) PollOutputsContainer(_ context.Context, ctn *pipeline.Container // PollFileNames grabs test results and attachments from provided path within a container. // This is a no-op for kubernetes. Pod environments cannot be dynamic. -func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, paths []string) ([]string, error) { +func (c *client) PollFileNames(_ context.Context, ctn *pipeline.Container, paths []string) ([]string, error) { c.Logger.Tracef("no-op: gathering test results and attachments from container %s", ctn.ID) return nil, nil @@ -87,7 +87,7 @@ func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, pat // PollFileContent captures the content and size of a file from the pipeline container. // This is a no-op for kubernetes. Pod environments cannot be dynamic. -func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, path string) (io.Reader, int64, error) { +func (c *client) PollFileContent(_ context.Context, ctn *pipeline.Container, path string) (io.Reader, int64, error) { c.Logger.Tracef("no-op: gathering test results and attachments from container %s", ctn.ID) return nil, 0, nil From 46db70b2492e255f85bb43d24636f7e8370429c1 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Tue, 30 Dec 2025 09:49:44 -0600 Subject: [PATCH 34/63] enhance: use context for API calls and refresh install token --- cmd/vela-worker/client.go | 5 ++- cmd/vela-worker/exec.go | 30 +++++++------- cmd/vela-worker/operate.go | 13 +++--- cmd/vela-worker/register.go | 24 +++++------ docker-compose.yml | 5 ++- executor/flags.go | 2 +- executor/linux/build.go | 67 ++++++++++++++++++++++++------- executor/linux/secret.go | 22 ++++++---- executor/linux/secret_test.go | 2 +- executor/linux/service.go | 41 +++++++++---------- executor/linux/stage.go | 7 +++- executor/linux/step.go | 45 +++++++++++---------- executor/local/build.go | 12 +++--- executor/local/service.go | 4 +- executor/local/step.go | 4 +- go.mod | 5 +-- go.sum | 10 ++--- internal/build/snapshot.go | 5 ++- internal/build/snapshot_test.go | 2 +- internal/build/upload.go | 5 ++- internal/build/upload_test.go | 2 +- internal/service/snapshot.go | 5 ++- internal/service/snapshot_test.go | 2 +- internal/service/upload.go | 5 ++- internal/service/upload_test.go | 2 +- internal/step/snapshot.go | 11 ++--- internal/step/snapshot_test.go | 4 +- internal/step/upload.go | 5 ++- internal/step/upload_test.go | 2 +- router/middleware/perm/perm.go | 2 +- 30 files changed, 202 insertions(+), 148 deletions(-) diff --git a/cmd/vela-worker/client.go b/cmd/vela-worker/client.go index 9f8a9031..b0f093ac 100644 --- a/cmd/vela-worker/client.go +++ b/cmd/vela-worker/client.go @@ -6,6 +6,7 @@ import ( "github.com/sirupsen/logrus" "github.com/go-vela/sdk-go/vela" + api "github.com/go-vela/server/api/types" ) // helper function to setup the vela API client for worker check-in and build executable retrieval. @@ -23,7 +24,7 @@ func setupClient(s *Server, token string) (*vela.Client, error) { } // helper function to setup the vela API client for executor calls to update build resources. -func setupExecClient(s *Server, buildToken, scmToken string) (*vela.Client, error) { +func setupExecClient(s *Server, buildToken, scmToken string, exp int64, build *api.Build) (*vela.Client, error) { logrus.Debug("creating vela client from worker configuration") vela, err := vela.NewClient(s.Address, "", nil) @@ -31,7 +32,7 @@ func setupExecClient(s *Server, buildToken, scmToken string) (*vela.Client, erro return nil, err } - vela.Authentication.SetBuildTokenAuth(buildToken, scmToken) + vela.Authentication.SetBuildTokenAuth(buildToken, scmToken, exp, build.GetRepo().GetFullName(), build.GetNumber()) return vela, nil } diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index 2ace5282..b789db48 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -27,7 +27,7 @@ import ( // and execute Vela pipelines for the Worker. // //nolint:gocyclo,funlen // ignore cyclomatic complexity and function length -func (w *Worker) exec(index int, config *api.Worker) error { +func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error { var err error var execStorage storage.Storage @@ -55,7 +55,7 @@ func (w *Worker) exec(index int, config *api.Worker) error { logrus.Debugf("queue item prep - attempt %d", i+1) // get worker from database - worker, _, err := w.VelaClient.Worker.Get(w.Config.API.Address.Hostname()) + worker, _, err := w.VelaClient.Worker.Get(ctx, w.Config.API.Address.Hostname()) if err != nil { logrus.Errorf("unable to retrieve worker from server: %s", err) @@ -71,7 +71,7 @@ func (w *Worker) exec(index int, config *api.Worker) error { // capture an item from the queue only on first loop iteration (failures here return nil) if i == 0 { - item, err = w.Queue.Pop(context.Background(), worker.GetRoutes()) + item, err = w.Queue.Pop(ctx, worker.GetRoutes()) if err != nil { logrus.Errorf("queue pop failed: %v", err) @@ -90,7 +90,7 @@ func (w *Worker) exec(index int, config *api.Worker) error { } // retrieve a build token from the server to setup the execBuildClient - bt, resp, err := w.VelaClient.Build.GetBuildToken(item.Build.GetRepo().GetOrg(), item.Build.GetRepo().GetName(), item.Build.GetNumber()) + bt, resp, err := w.VelaClient.Build.GetBuildToken(ctx, item.Build.GetRepo().GetOrg(), item.Build.GetRepo().GetName(), item.Build.GetNumber()) if err != nil { logrus.Errorf("unable to retrieve build token: %s", err) @@ -125,7 +125,7 @@ func (w *Worker) exec(index int, config *api.Worker) error { } // request build executable containing pipeline.Build data using exec client - execBuildExecutable, _, err = execBuildClient.Build.GetBuildExecutable(item.Build.GetRepo().GetOrg(), item.Build.GetRepo().GetName(), item.Build.GetNumber()) + execBuildExecutable, _, err = execBuildClient.Build.GetBuildExecutable(ctx, item.Build.GetRepo().GetOrg(), item.Build.GetRepo().GetName(), item.Build.GetNumber()) if err != nil { // check if the retry limit has been exceeded if i < retries-1 { @@ -146,8 +146,10 @@ func (w *Worker) exec(index int, config *api.Worker) error { return err } + logrus.Debugf("setting up exec client with scm token %s with expiration %d", p.Token, p.TokenExp) + // setup exec client with scm token and build token - execBuildClient, err = setupExecClient(w.Config.Server, bt.GetToken(), p.Token) + execBuildClient, err = setupExecClient(w.Config.Server, bt.GetToken(), p.Token, p.TokenExp, item.Build) if err != nil { return err } @@ -198,7 +200,7 @@ func (w *Worker) exec(index int, config *api.Worker) error { config.SetLastBuildStartedAt(time.Now().Unix()) // update worker in the database - _, _, err = w.VelaClient.Worker.Update(config.GetHostname(), config) + _, _, err = w.VelaClient.Worker.Update(ctx, config.GetHostname(), config) if err != nil { logger.Errorf("unable to update worker: %v", err) } @@ -215,7 +217,7 @@ func (w *Worker) exec(index int, config *api.Worker) error { build.SetStatus(constants.StatusError) build.SetFinished(time.Now().UTC().Unix()) - _, _, err := execBuildClient.Build.Update(build) + _, _, err := execBuildClient.Build.Update(ctx, build) if err != nil { logrus.Errorf("Unable to set build status to %s: %s", constants.StatusFailure, err) return err @@ -288,9 +290,8 @@ func (w *Worker) exec(index int, config *api.Worker) error { logger.Info("destroying build") - // destroy the build with the executor (pass a background - // context to guarantee all build resources are destroyed). - err = _executor.DestroyBuild(context.Background()) + // destroy the build with the executor + err = _executor.DestroyBuild(ctx) if err != nil { logger.Errorf("unable to destroy build: %v", err) } @@ -317,7 +318,7 @@ func (w *Worker) exec(index int, config *api.Worker) error { config.SetLastBuildFinishedAt(time.Now().Unix()) // update worker in the database - _, _, err := w.VelaClient.Worker.Update(config.GetHostname(), config) + _, _, err := w.VelaClient.Worker.Update(ctx, config.GetHostname(), config) if err != nil { logger.Errorf("unable to update worker: %v", err) } @@ -331,9 +332,8 @@ func (w *Worker) exec(index int, config *api.Worker) error { t = time.Duration(item.Build.GetRepo().GetTimeout()) * time.Minute } - // create a build context (from a background context - // so that other builds can't inadvertently cancel this build) - buildCtx, done := context.WithCancel(context.Background()) + // create a build context + buildCtx, done := context.WithCancel(ctx) defer done() // add to the background context with a timeout diff --git a/cmd/vela-worker/operate.go b/cmd/vela-worker/operate.go index 5085f8b4..74f5272d 100644 --- a/cmd/vela-worker/operate.go +++ b/cmd/vela-worker/operate.go @@ -54,7 +54,7 @@ func (w *Worker) operate(ctx context.Context) error { logrus.Trace("getting queue creds") // fetching queue credentials using registration token - creds, _, err := w.VelaClient.Queue.GetInfo() + creds, _, err := w.VelaClient.Queue.GetInfo(ctx) if err != nil { logrus.Trace("error getting creds") return err @@ -75,7 +75,7 @@ func (w *Worker) operate(ctx context.Context) error { if err != nil { logrus.Error("queue setup failed") // set to error as queue setup fails - w.updateWorkerStatus(registryWorker, constants.WorkerStatusError) + w.updateWorkerStatus(ctx, registryWorker, constants.WorkerStatusError) } // getting storage creds @@ -148,7 +148,7 @@ func (w *Worker) operate(ctx context.Context) error { // check in attempt loop for { // register or update the worker - w.CheckedIn, token, err = w.checkIn(registryWorker) + w.CheckedIn, token, err = w.checkIn(ctx, registryWorker) // check in failed if err != nil { // check if token is expired @@ -259,17 +259,16 @@ func (w *Worker) operate(ctx context.Context) error { }).Info("running worker executor exec") // exec operator subprocess to poll and execute builds - // (do not pass the context to avoid errors in one + // (pass background context to avoid errors in one // executor+build inadvertently canceling other builds) - //nolint:contextcheck // ignore passing context - err = w.exec(id, registryWorker) + err = w.exec(context.Background(), id, registryWorker) if err != nil { // log the error received from the executor // // https://pkg.go.dev/github.com/sirupsen/logrus#Errorf logrus.Errorf("failing worker executor: %v", err) registryWorker.SetStatus(constants.WorkerStatusError) - _, resp, logErr := w.VelaClient.Worker.Update(registryWorker.GetHostname(), registryWorker) + _, resp, logErr := w.VelaClient.Worker.Update(ctx, registryWorker.GetHostname(), registryWorker) if resp == nil { // log the error instead of returning so the operation doesn't block worker deployment diff --git a/cmd/vela-worker/register.go b/cmd/vela-worker/register.go index ac0deabc..eef60910 100644 --- a/cmd/vela-worker/register.go +++ b/cmd/vela-worker/register.go @@ -15,7 +15,7 @@ import ( ) // checkIn is a helper function to phone home to the server. -func (w *Worker) checkIn(config *api.Worker) (bool, string, error) { +func (w *Worker) checkIn(ctx context.Context, config *api.Worker) (bool, string, error) { // check to see if the worker already exists in the database logrus.Infof("retrieving worker %s from the server", config.GetHostname()) @@ -32,7 +32,7 @@ func (w *Worker) checkIn(config *api.Worker) (bool, string, error) { time.Sleep(time.Duration(i*10) * time.Second) } - _, resp, err := w.VelaClient.Worker.Get(config.GetHostname()) + _, resp, err := w.VelaClient.Worker.Get(ctx, config.GetHostname()) if err != nil { respErr := fmt.Errorf("unable to retrieve worker %s from the server: %w", config.GetHostname(), err) // if server is down, the worker status will not be updated @@ -41,7 +41,7 @@ func (w *Worker) checkIn(config *api.Worker) (bool, string, error) { } // if we receive a 404 the worker needs to be registered if resp.StatusCode == http.StatusNotFound { - registered, strToken, regErr := w.register(config) + registered, strToken, regErr := w.register(ctx, config) if regErr != nil { if i < retries-1 { logrus.WithError(err).Warningf("retrying #%d", i+1) @@ -58,7 +58,7 @@ func (w *Worker) checkIn(config *api.Worker) (bool, string, error) { // if we were able to GET the worker, update it logrus.Infof("checking worker %s into the server", config.GetHostname()) - tkn, _, err = w.VelaClient.Worker.RefreshAuth(config.GetHostname()) + tkn, _, err = w.VelaClient.Worker.RefreshAuth(ctx, config.GetHostname()) if err != nil { if i < retries-1 { logrus.WithError(err).Warningf("retrying #%d", i+1) @@ -68,7 +68,7 @@ func (w *Worker) checkIn(config *api.Worker) (bool, string, error) { } // set to error when check in fails - w.updateWorkerStatus(config, constants.WorkerStatusError) + w.updateWorkerStatus(ctx, config, constants.WorkerStatusError) return false, "", fmt.Errorf("unable to refresh auth for worker %s on the server: %w", config.GetHostname(), err) } @@ -76,7 +76,7 @@ func (w *Worker) checkIn(config *api.Worker) (bool, string, error) { status := w.getWorkerStatusFromConfig(config) // update worker status to Idle when checkIn is successful. - w.updateWorkerStatus(config, status) + w.updateWorkerStatus(ctx, config, status) break } @@ -85,14 +85,14 @@ func (w *Worker) checkIn(config *api.Worker) (bool, string, error) { } // register is a helper function to register the worker with the server. -func (w *Worker) register(config *api.Worker) (bool, string, error) { +func (w *Worker) register(ctx context.Context, config *api.Worker) (bool, string, error) { logrus.Infof("worker %s not found, registering it with the server", config.GetHostname()) // status Idle will be set for worker upon first time registration // if worker cannot be registered, no status will be set. config.SetStatus(constants.WorkerStatusIdle) - tkn, _, err := w.VelaClient.Worker.Add(config) + tkn, _, err := w.VelaClient.Worker.Add(ctx, config) if err != nil { // log the error instead of returning so the operation doesn't block worker deployment return false, "", fmt.Errorf("unable to register worker %s with the server: %w", config.GetHostname(), err) @@ -110,7 +110,7 @@ func (w *Worker) queueCheckIn(ctx context.Context, registryWorker *api.Worker) ( if pErr != nil { logrus.Errorf("worker %s unable to contact the queue: %v", registryWorker.GetHostname(), pErr) // set status to error as queue is not available - w.updateWorkerStatus(registryWorker, constants.WorkerStatusError) + w.updateWorkerStatus(ctx, registryWorker, constants.WorkerStatusError) return false, pErr } @@ -118,16 +118,16 @@ func (w *Worker) queueCheckIn(ctx context.Context, registryWorker *api.Worker) ( status := w.getWorkerStatusFromConfig(registryWorker) // update worker status to Idle when setup and ping are good. - w.updateWorkerStatus(registryWorker, status) + w.updateWorkerStatus(ctx, registryWorker, status) return true, nil } // updateWorkerStatus is a helper function to update worker status // logs the error if it can't update status. -func (w *Worker) updateWorkerStatus(config *api.Worker, status string) { +func (w *Worker) updateWorkerStatus(ctx context.Context, config *api.Worker, status string) { config.SetStatus(status) - _, resp, logErr := w.VelaClient.Worker.Update(config.GetHostname(), config) + _, resp, logErr := w.VelaClient.Worker.Update(ctx, config.GetHostname(), config) if resp == nil { // log the error instead of returning so the operation doesn't block worker deployment diff --git a/docker-compose.yml b/docker-compose.yml index 17722ff6..1436bc03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,10 +51,13 @@ services: # https://go-vela.github.io/docs/administration/server/ server: container_name: server - image: target/vela-server:latest + image: server:local networks: - vela environment: + CACHE_DRIVER: redis + CACHE_ADDR: 'redis://redis:6379' + CACHE_INSTALL_TOKEN_KEY: df4b22849583a95cf1d581f234e3249f0ab96bc69ca9dc90da3dfaff173d8b6a DATABASE_DRIVER: postgres DATABASE_ADDR: 'postgres://vela:zB7mrKDTZqNeNTD8z47yG4DHywspAh@postgres:5432/vela?sslmode=disable' DATABASE_COMPRESSION_LEVEL: 3 diff --git a/executor/flags.go b/executor/flags.go index c3d53cf1..ffd70d90 100644 --- a/executor/flags.go +++ b/executor/flags.go @@ -40,7 +40,7 @@ var Flags = []cli.Flag{ Name: "executor.log_streaming_timeout", Usage: "maximum amount of time to wait for log streaming after build completes", Sources: cli.EnvVars("WORKER_LOG_STREAMING_TIMEOUT", "VELA_LOG_STREAMING_TIMEOUT", "LOG_STREAMING_TIMEOUT"), - Value: 5 * time.Minute, + Value: 1 * time.Minute, }, &cli.BoolFlag{ Name: "executor.enforce-trusted-repos", diff --git a/executor/linux/build.go b/executor/linux/build.go index 859c000e..487bde34 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -28,7 +28,7 @@ func (c *client) CreateBuild(ctx context.Context) error { // defer taking a snapshot of the build // // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Snapshot - defer func() { build.Snapshot(c.build, c.Vela, c.err, c.Logger) }() + defer func() { build.Snapshot(ctx, c.build, c.Vela, c.err, c.Logger) }() // update the build fields c.build.SetStatus(constants.StatusRunning) @@ -41,7 +41,7 @@ func (c *client) CreateBuild(ctx context.Context) error { // send API call to update the build // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#BuildService.Update - c.build, _, c.err = c.Vela.Build.Update(c.build) + c.build, _, c.err = c.Vela.Build.Update(ctx, c.build) if c.err != nil { return fmt.Errorf("unable to upload build state: %w", c.err) } @@ -82,7 +82,9 @@ func (c *client) PlanBuild(ctx context.Context) error { // defer taking a snapshot of the build // // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Snapshot - defer func() { build.Snapshot(c.build, c.Vela, c.err, c.Logger) }() + // + //nolint:contextcheck // ctx can be canceled by build timing out. need to pass background context here + defer func() { build.Snapshot(context.Background(), c.build, c.Vela, c.err, c.Logger) }() // load the init step from the client // @@ -106,12 +108,14 @@ func (c *client) PlanBuild(ctx context.Context) error { // defer taking a snapshot of the init step // // https://pkg.go.dev/github.com/go-vela/worker/internal/step#SnapshotInit + // + //nolint:contextcheck // ctx can be canceled by build timing out. need to pass background context here defer func() { if c.err != nil { _init.SetStatus(constants.StatusFailure) } - step.SnapshotInit(c.init, c.build, c.Vela, c.Logger, _init, _log) + step.SnapshotInit(context.Background(), c.init, c.build, c.Vela, c.Logger, _init, _log) }() c.Logger.Info("creating network") @@ -173,7 +177,7 @@ func (c *client) PlanBuild(ctx context.Context) error { c.Logger.Infof("pulling secret: %s", secret.Name) - s, err := c.secret.pull(secret) + s, err := c.secret.pull(ctx, secret) if err != nil { c.err = err return fmt.Errorf("unable to pull secrets: %w", err) @@ -212,7 +216,9 @@ func (c *client) AssembleBuild(ctx context.Context) error { // defer taking a snapshot of the build // // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Snapshot - defer func() { build.Snapshot(c.build, c.Vela, c.err, c.Logger) }() + // + //nolint:contextcheck // ctx can be canceled by build timing out. need to pass background context here + defer func() { build.Snapshot(context.Background(), c.build, c.Vela, c.err, c.Logger) }() // load the init step from the client // @@ -238,15 +244,16 @@ func (c *client) AssembleBuild(ctx context.Context) error { _init.SetStatus(constants.StatusFailure) } - step.Upload(c.init, c.build, c.Vela, c.Logger, _init) + step.Upload(ctx, c.init, c.build, c.Vela, c.Logger, _init) }() + //nolint:contextcheck // ctx can be canceled by build timing out. need to pass background context here defer func() { c.Logger.Infof("uploading %s step logs", c.init.Name) // send API call to update the logs for the step // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#LogService.UpdateStep - _, err = c.Vela.Log.UpdateStep(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), c.init.Number, _log) + _, err = c.Vela.Log.UpdateStep(context.Background(), c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), c.init.Number, _log) if err != nil { c.Logger.Errorf("unable to upload %s logs: %v", c.init.Name, err) } @@ -366,7 +373,7 @@ func (c *client) AssembleBuild(ctx context.Context) error { Commands: len(s.Origin.Commands) > 0 || len(s.Origin.Entrypoint) > 0, } - tkn, _, err := c.Vela.Build.GetIDRequestToken(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), opts) + tkn, _, err := c.Vela.Build.GetIDRequestToken(ctx, c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), opts) if err != nil { return err } @@ -427,6 +434,7 @@ func (c *client) AssembleBuild(ctx context.Context) error { // //nolint:funlen // there is a lot going on here and will probably always be long func (c *client) ExecBuild(ctx context.Context) error { + //nolint:contextcheck // ctx can be canceled by build timing out. need to pass background context here defer func() { // Exec* calls are responsible for sending StreamRequest messages. // close the channel at the end of ExecBuild to signal that @@ -436,7 +444,7 @@ func (c *client) ExecBuild(ctx context.Context) error { // defer an upload of the build // // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Upload - build.Upload(c.build, c.Vela, c.err, c.Logger) + build.Upload(context.Background(), c.build, c.Vela, c.err, c.Logger) }() // output maps for dynamic environment variables captured from volume @@ -473,6 +481,11 @@ func (c *client) ExecBuild(ctx context.Context) error { return fmt.Errorf("unable to plan service: %w", c.err) } + c.err = c.UpdateSCMAuth(ctx, _service) + if c.err != nil { + return fmt.Errorf("unable to update SCM auth: %w", c.err) + } + c.Logger.Infof("executing %s service", _service.Name) // execute the service c.err = c.ExecService(ctx, _service) @@ -487,6 +500,11 @@ func (c *client) ExecBuild(ctx context.Context) error { continue } + c.err = c.UpdateSCMAuth(ctx, _step) + if c.err != nil { + return fmt.Errorf("unable to update SCM auth: %w", c.err) + } + // poll outputs opEnv, maskEnv, c.err = c.outputs.poll(ctx, c.OutputCtn) if c.err != nil { @@ -583,7 +601,7 @@ func (c *client) ExecBuild(ctx context.Context) error { _step.Secrets = append(_step.Secrets, sec) // load any lazy secrets into the container environment - c.err = loadLazySecrets(c, _step) + c.err = loadLazySecrets(ctx, c, _step) if c.err != nil { return fmt.Errorf("unable to plan step: %w", c.err) } @@ -791,7 +809,7 @@ func (c *client) StreamBuild(ctx context.Context) error { // during build planning. It is only available for the Docker runtime. // -func loadLazySecrets(c *client, _step *pipeline.Container) error { +func loadLazySecrets(ctx context.Context, c *client, _step *pipeline.Container) error { _log := new(api.Log) lazySecrets := make(map[string]*api.Secret) @@ -861,7 +879,7 @@ func loadLazySecrets(c *client, _step *pipeline.Container) error { _log.AppendData([]byte( fmt.Sprintf("unable to pull secret %s: lazy loading secrets not available with Kubernetes runtime\n", s.Source))) - _, err := c.Vela.Log.UpdateStep(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), _step.Number, _log) + _, err := c.Vela.Log.UpdateStep(ctx, c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), _step.Number, _log) if err != nil { return err } @@ -871,7 +889,7 @@ func loadLazySecrets(c *client, _step *pipeline.Container) error { c.Logger.Infof("pulling secret %s", secret.Name) - s, err := c.secret.pull(secret) + s, err := c.secret.pull(ctx, secret) if err != nil { c.err = err return fmt.Errorf("unable to pull secrets: %w", err) @@ -889,7 +907,7 @@ func loadLazySecrets(c *client, _step *pipeline.Container) error { _log.AppendData(append(sRaw, "\n"...)) - _, err = c.Vela.Log.UpdateStep(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), _step.Number, _log) + _, err = c.Vela.Log.UpdateStep(ctx, c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), _step.Number, _log) if err != nil { return err } @@ -1044,3 +1062,22 @@ func (c *client) DestroyBuild(ctx context.Context) error { return err } + +func (c *client) UpdateSCMAuth(ctx context.Context, ctn *pipeline.Container) error { + // refresh SCM token if within 45 minutes of expiration + // + // this is an arbitrary range. We want the installation token to always be fresh for steps but don't want to refresh for every step. + if c.Vela.Authentication.SCMExpiration() != 0 && time.Now().Unix() >= c.Vela.Authentication.SCMExpiration()-int64((45*time.Minute).Seconds()) { + c.Logger.Info("refreshing SCM token") + + _, err := c.Vela.Authentication.RefreshInstallToken(ctx, c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber()) + if err != nil { + return fmt.Errorf("unable to refresh SCM token: %w", err) + } + + ctn.Environment["VELA_NETRC_PASSWORD"] = c.Vela.Authentication.SCMToken() + ctn.Environment["VELA_GIT_TOKEN"] = c.Vela.Authentication.SCMToken() + } + + return nil +} diff --git a/executor/linux/secret.go b/executor/linux/secret.go index c938615b..7e375217 100644 --- a/executor/linux/secret.go +++ b/executor/linux/secret.go @@ -118,6 +118,7 @@ func (s *secretSvc) exec(ctx context.Context, p *pipeline.SecretSlice) error { return err } + //nolint:contextcheck // ctx can be canceled by build timing out defer func() { _init.SetFinished(time.Now().UTC().Unix()) @@ -125,7 +126,7 @@ func (s *secretSvc) exec(ctx context.Context, p *pipeline.SecretSlice) error { // send API call to update the build // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#StepService.Update - _, _, err = s.client.Vela.Step.Update(s.client.build.GetRepo().GetOrg(), s.client.build.GetRepo().GetName(), s.client.build.GetNumber(), _init) + _, _, err = s.client.Vela.Step.Update(context.Background(), s.client.build.GetRepo().GetOrg(), s.client.build.GetRepo().GetName(), s.client.build.GetNumber(), _init) if err != nil { s.client.Logger.Errorf("unable to upload init state: %v", err) } @@ -140,6 +141,11 @@ func (s *secretSvc) exec(ctx context.Context, p *pipeline.SecretSlice) error { continue } + s.client.err = s.client.UpdateSCMAuth(ctx, _secret.Origin) + if s.client.err != nil { + return fmt.Errorf("unable to update SCM auth: %w", s.client.err) + } + opEnv, maskEnv, s.client.err = s.client.outputs.poll(ctx, s.client.OutputCtn) if s.client.err != nil { return fmt.Errorf("unable to exec outputs container: %w", s.client.err) @@ -209,7 +215,7 @@ func (s *secretSvc) exec(ctx context.Context, p *pipeline.SecretSlice) error { // send API call to update the build // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#StepService.Update - _, _, err = s.client.Vela.Step.Update(s.client.build.GetRepo().GetOrg(), s.client.build.GetRepo().GetName(), s.client.build.GetNumber(), _init) + _, _, err = s.client.Vela.Step.Update(ctx, s.client.build.GetRepo().GetOrg(), s.client.build.GetRepo().GetName(), s.client.build.GetNumber(), _init) if err != nil { s.client.Logger.Errorf("unable to upload init state: %v", err) } @@ -219,7 +225,7 @@ func (s *secretSvc) exec(ctx context.Context, p *pipeline.SecretSlice) error { } // pull defines a function that pulls the secrets from the server for a given pipeline. -func (s *secretSvc) pull(secret *pipeline.Secret) (*api.Secret, error) { +func (s *secretSvc) pull(ctx context.Context, secret *pipeline.Secret) (*api.Secret, error) { _secret := new(api.Secret) switch secret.Type { @@ -233,7 +239,7 @@ func (s *secretSvc) pull(secret *pipeline.Secret) (*api.Secret, error) { // send API call to capture the org secret // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#SecretService.Get - _secret, _, err = s.client.Vela.Secret.Get(secret.Engine, secret.Type, org, "*", key) + _secret, _, err = s.client.Vela.Secret.Get(ctx, secret.Engine, secret.Type, org, "*", key) if err != nil { return nil, fmt.Errorf("%w: %w", ErrUnableToRetrieve, err) } @@ -250,7 +256,7 @@ func (s *secretSvc) pull(secret *pipeline.Secret) (*api.Secret, error) { // send API call to capture the repo secret // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#SecretService.Get - _secret, _, err = s.client.Vela.Secret.Get(secret.Engine, secret.Type, org, repo, key) + _secret, _, err = s.client.Vela.Secret.Get(ctx, secret.Engine, secret.Type, org, repo, key) if err != nil { return nil, fmt.Errorf("%w: %w", ErrUnableToRetrieve, err) } @@ -267,7 +273,7 @@ func (s *secretSvc) pull(secret *pipeline.Secret) (*api.Secret, error) { // send API call to capture the repo secret // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#SecretService.Get - _secret, _, err = s.client.Vela.Secret.Get(secret.Engine, secret.Type, org, team, key) + _secret, _, err = s.client.Vela.Secret.Get(ctx, secret.Engine, secret.Type, org, team, key) if err != nil { return nil, fmt.Errorf("%w: %w", ErrUnableToRetrieve, err) } @@ -310,7 +316,7 @@ func (s *secretSvc) stream(ctx context.Context, ctn *pipeline.Container) error { // send API call to update the logs for the service // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#LogService.UpdateService - _, err = s.client.Vela.Log.UpdateStep(s.client.build.GetRepo().GetOrg(), s.client.build.GetRepo().GetName(), s.client.build.GetNumber(), s.client.init.Number, _log) + _, err = s.client.Vela.Log.UpdateStep(ctx, s.client.build.GetRepo().GetOrg(), s.client.build.GetRepo().GetName(), s.client.build.GetNumber(), s.client.init.Number, _log) if err != nil { logger.Errorf("unable to upload container logs: %v", err) } @@ -343,7 +349,7 @@ func (s *secretSvc) stream(ctx context.Context, ctn *pipeline.Container) error { // send API call to append the logs for the init step // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#LogService.UpdateStep - _, err = s.client.Vela.Log.UpdateStep(s.client.build.GetRepo().GetOrg(), s.client.build.GetRepo().GetName(), s.client.build.GetNumber(), s.client.init.Number, _log) + _, err = s.client.Vela.Log.UpdateStep(ctx, s.client.build.GetRepo().GetOrg(), s.client.build.GetRepo().GetName(), s.client.build.GetNumber(), s.client.init.Number, _log) if err != nil { return err } diff --git a/executor/linux/secret_test.go b/executor/linux/secret_test.go index b8844c11..7b8efdfa 100644 --- a/executor/linux/secret_test.go +++ b/executor/linux/secret_test.go @@ -787,7 +787,7 @@ func TestLinux_Secret_pull(t *testing.T) { t.Errorf("unable to create %s executor engine: %v", test.name, err) } - _, err = _engine.secret.pull(test.secret) + _, err = _engine.secret.pull(t.Context(), test.secret) if test.failure { if err == nil { diff --git a/executor/linux/service.go b/executor/linux/service.go index e60b2e97..ec309aa2 100644 --- a/executor/linux/service.go +++ b/executor/linux/service.go @@ -65,7 +65,7 @@ func (c *client) CreateService(ctx context.Context, ctn *pipeline.Container) err } // PlanService prepares the service for execution. -func (c *client) PlanService(_ context.Context, ctn *pipeline.Container) error { +func (c *client) PlanService(ctx context.Context, ctn *pipeline.Container) error { var err error // update engine logger with service metadata // @@ -87,7 +87,7 @@ func (c *client) PlanService(_ context.Context, ctn *pipeline.Container) error { // send API call to update the service // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#SvcService.Update - _service, _, err = c.Vela.Svc.Update(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), _service) + _service, _, err = c.Vela.Svc.Update(ctx, c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), _service) if err != nil { return err } @@ -108,7 +108,7 @@ func (c *client) PlanService(_ context.Context, ctn *pipeline.Container) error { // send API call to capture the service log // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#LogService.GetService - _log, _, err := c.Vela.Log.GetService(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), _service.GetNumber()) + _log, _, err := c.Vela.Log.GetService(ctx, c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), _service.GetNumber()) if err != nil { return err } @@ -137,7 +137,20 @@ func (c *client) ExecService(ctx context.Context, ctn *pipeline.Container) error // defer taking a snapshot of the service // // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Snapshot - defer func() { service.Snapshot(ctn, c.build, c.Vela, c.Logger, _service) }() + defer func() { + // if context has been canceled, do a final upload for service + if err := ctx.Err(); err != nil { + logger.Debug("context exceeded during service execution, marking service as failure") + _service.SetStatus(constants.StatusFailure) + + //nolint:contextcheck // ctx canceled. need to pass background context here + service.Upload(context.Background(), ctn, c.build, c.Vela, c.Logger, _service) + + return + } + + service.Snapshot(ctx, ctn, c.build, c.Vela, c.Logger, _service) + }() // verify service is allowed to run if c.enforceTrustedRepos { @@ -218,7 +231,7 @@ func (c *client) StreamService(ctx context.Context, ctn *pipeline.Container) err // send API call to update the logs for the service // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#LogService.UpdateService - _, err = c.Vela.Log.UpdateService(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), ctn.Number, _log) + _, err = c.Vela.Log.UpdateService(ctx, c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), ctn.Number, _log) if err != nil { logger.Errorf("unable to upload container logs: %v", err) } @@ -275,7 +288,7 @@ func (c *client) StreamService(ctx context.Context, ctn *pipeline.Container) err // send API call to append the logs for the service // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#LogService.UpdateService - _, err = c.Vela.Log.UpdateService(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), ctn.Number, _log) + _, err = c.Vela.Log.UpdateService(ctx, c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), ctn.Number, _log) if err != nil { logger.Error(err) } @@ -318,23 +331,9 @@ func (c *client) DestroyService(ctx context.Context, ctn *pipeline.Container) er // https://pkg.go.dev/github.com/sirupsen/logrus#Entry.WithField logger := c.Logger.WithField("service", ctn.Name) - // load the service from the client - // - // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Load - _service, err := service.Load(ctn, &c.services) - if err != nil { - // create the service from the container - _service = api.ServiceFromContainerEnvironment(ctn) - } - - // defer an upload of the service - // - // https://pkg.go.dev/github.com/go-vela/worker/internal/service#LoaUploadd - defer func() { service.Upload(ctn, c.build, c.Vela, logger, _service) }() - logger.Debug("inspecting container") // inspect the runtime container - err = c.Runtime.InspectContainer(ctx, ctn) + err := c.Runtime.InspectContainer(ctx, ctn) if err != nil { return err } diff --git a/executor/linux/stage.go b/executor/linux/stage.go index 20b25982..899afcd9 100644 --- a/executor/linux/stage.go +++ b/executor/linux/stage.go @@ -141,6 +141,11 @@ func (c *client) ExecStage(ctx context.Context, s *pipeline.Stage, m *sync.Map) continue } + c.err = c.UpdateSCMAuth(ctx, _step) + if c.err != nil { + return fmt.Errorf("unable to update SCM auth: %w", c.err) + } + // add netrc to secrets for masking in logs sec := &pipeline.StepSecret{ Target: "VELA_NETRC_PASSWORD", @@ -148,7 +153,7 @@ func (c *client) ExecStage(ctx context.Context, s *pipeline.Stage, m *sync.Map) _step.Secrets = append(_step.Secrets, sec) // load any lazy secrets and inject them into container environment - err = loadLazySecrets(c, _step) + err = loadLazySecrets(ctx, c, _step) if err != nil { return fmt.Errorf("unable to plan step %s: %w", _step.Name, err) } diff --git a/executor/linux/step.go b/executor/linux/step.go index 5e4eaa56..b0a83fde 100644 --- a/executor/linux/step.go +++ b/executor/linux/step.go @@ -78,7 +78,7 @@ func (c *client) CreateStep(ctx context.Context, ctn *pipeline.Container) error } // PlanStep prepares the step for execution. -func (c *client) PlanStep(_ context.Context, ctn *pipeline.Container) error { +func (c *client) PlanStep(ctx context.Context, ctn *pipeline.Container) error { var err error // update engine logger with step metadata @@ -95,7 +95,7 @@ func (c *client) PlanStep(_ context.Context, ctn *pipeline.Container) error { // send API call to update the step // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#StepService.Update - _step, _, err = c.Vela.Step.Update(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), _step) + _step, _, err = c.Vela.Step.Update(ctx, c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), _step) if err != nil { return err } @@ -109,7 +109,7 @@ func (c *client) PlanStep(_ context.Context, ctn *pipeline.Container) error { Commands: len(ctn.Commands) > 0 || len(ctn.Entrypoint) > 0, } - tkn, _, err := c.Vela.Build.GetIDRequestToken(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), opts) + tkn, _, err := c.Vela.Build.GetIDRequestToken(ctx, c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), opts) if err != nil { return err } @@ -133,7 +133,7 @@ func (c *client) PlanStep(_ context.Context, ctn *pipeline.Container) error { // send API call to capture the step log // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#LogService.GetStep - _log, _, err := c.Vela.Log.GetStep(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), _step.GetNumber()) + _log, _, err := c.Vela.Log.GetStep(ctx, c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), _step.GetNumber()) if err != nil { return err } @@ -166,7 +166,20 @@ func (c *client) ExecStep(ctx context.Context, ctn *pipeline.Container) error { // defer taking a snapshot of the step // // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Snapshot - defer func() { step.Snapshot(ctn, c.build, c.Vela, c.Logger, _step) }() + defer func() { + // if context has been canceled, do a final upload for step + if err := ctx.Err(); err != nil { + logger.Debug("context exceeded during step execution, marking step as failure") + _step.SetStatus(constants.StatusFailure) + + //nolint:contextcheck // ctx canceled. need to pass background context here + step.Upload(context.Background(), ctn, c.build, c.Vela, c.Logger, _step) + + return + } + + step.Snapshot(ctx, ctn, c.build, c.Vela, c.Logger, _step) + }() // verify step is allowed to run if c.enforceTrustedRepos { @@ -282,7 +295,9 @@ func (c *client) StreamStep(ctx context.Context, ctn *pipeline.Container) error // send API call to update the logs for the step // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#LogService.UpdateStep - _, err = c.Vela.Log.UpdateStep(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), ctn.Number, _log) + // + //nolint:contextcheck // ctx can be canceled by build timing out. need to pass background context here + _, err = c.Vela.Log.UpdateStep(context.Background(), c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), ctn.Number, _log) if err != nil { logger.Errorf("unable to upload container logs: %v", err) } @@ -342,7 +357,7 @@ func (c *client) StreamStep(ctx context.Context, ctn *pipeline.Container) error // send API call to append the logs for the step // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#LogStep.UpdateStep - _, err := c.Vela.Log.UpdateStep(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), ctn.Number, _log) + _, err := c.Vela.Log.UpdateStep(ctx, c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), ctn.Number, _log) if err != nil { logger.Error(err) } @@ -389,23 +404,9 @@ func (c *client) DestroyStep(ctx context.Context, ctn *pipeline.Container) error // https://pkg.go.dev/github.com/sirupsen/logrus#Entry.WithField logger := c.Logger.WithField("step", ctn.Name) - // load the step from the client - // - // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Load - _step, err := step.Load(ctn, &c.steps) - if err != nil { - // create the step from the container - _step = api.StepFromContainerEnvironment(ctn) - } - - // defer an upload of the step - // - // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Upload - defer func() { step.Upload(ctn, c.build, c.Vela, logger, _step) }() - logger.Debug("inspecting container") // inspect the runtime container - err = c.Runtime.InspectContainer(ctx, ctn) + err := c.Runtime.InspectContainer(ctx, ctn) if err != nil { return err } diff --git a/executor/local/build.go b/executor/local/build.go index bd083d24..d6c65a3c 100644 --- a/executor/local/build.go +++ b/executor/local/build.go @@ -22,7 +22,7 @@ func (c *client) CreateBuild(ctx context.Context) error { // defer taking a snapshot of the build // // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Snapshot - defer func() { build.Snapshot(c.build, nil, c.err, nil) }() + defer func() { build.Snapshot(ctx, c.build, nil, c.err, nil) }() // update the build fields c.build.SetStatus(constants.StatusRunning) @@ -65,7 +65,7 @@ func (c *client) PlanBuild(ctx context.Context) error { // defer taking a snapshot of the build // // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Snapshot - defer func() { build.Snapshot(c.build, nil, c.err, nil) }() + defer func() { build.Snapshot(ctx, c.build, nil, c.err, nil) }() // load the init step from the client // @@ -78,7 +78,7 @@ func (c *client) PlanBuild(ctx context.Context) error { // defer taking a snapshot of the init step // // https://pkg.go.dev/github.com/go-vela/worker/internal/step#SnapshotInit - defer func() { step.SnapshotInit(c.init, c.build, nil, nil, _init, nil) }() + defer func() { step.SnapshotInit(ctx, c.init, c.build, nil, nil, _init, nil) }() // create a step pattern for log output _pattern := fmt.Sprintf(stepPattern, c.init.Name) @@ -136,7 +136,7 @@ func (c *client) AssembleBuild(ctx context.Context) error { // defer taking a snapshot of the build // // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Snapshot - defer func() { build.Snapshot(c.build, nil, c.err, nil) }() + defer func() { build.Snapshot(ctx, c.build, nil, c.err, nil) }() // load the init step from the client // @@ -149,7 +149,7 @@ func (c *client) AssembleBuild(ctx context.Context) error { // defer an upload of the init step // // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Upload - defer func() { step.Upload(c.init, c.build, nil, nil, _init) }() + defer func() { step.Upload(ctx, c.init, c.build, nil, nil, _init) }() // create a step pattern for log output _pattern := fmt.Sprintf(stepPattern, c.init.Name) @@ -254,7 +254,7 @@ func (c *client) ExecBuild(ctx context.Context) error { // defer an upload of the build // // https://pkg.go.dev/github.com/go-vela/worker/internal/build#Upload - defer func() { build.Upload(c.build, nil, c.err, nil) }() + defer func() { build.Upload(ctx, c.build, nil, c.err, nil) }() // output maps for dynamic environment variables captured from volume var opEnv, maskEnv map[string]string diff --git a/executor/local/service.go b/executor/local/service.go index a7affb90..aded9ace 100644 --- a/executor/local/service.go +++ b/executor/local/service.go @@ -83,7 +83,7 @@ func (c *client) ExecService(ctx context.Context, ctn *pipeline.Container) error // defer taking a snapshot of the service // // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Snapshot - defer func() { service.Snapshot(ctn, c.build, nil, nil, _service) }() + defer func() { service.Snapshot(ctx, ctn, c.build, nil, nil, _service) }() // run the runtime container err = c.Runtime.RunContainer(ctx, ctn, c.pipeline) @@ -139,7 +139,7 @@ func (c *client) DestroyService(ctx context.Context, ctn *pipeline.Container) er // defer an upload of the service // // https://pkg.go.dev/github.com/go-vela/worker/internal/service#Upload - defer func() { service.Upload(ctn, c.build, nil, nil, _service) }() + defer func() { service.Upload(ctx, ctn, c.build, nil, nil, _service) }() // inspect the runtime container err = c.Runtime.InspectContainer(ctx, ctn) diff --git a/executor/local/step.go b/executor/local/step.go index cd1e1ffc..2287cb6a 100644 --- a/executor/local/step.go +++ b/executor/local/step.go @@ -82,7 +82,7 @@ func (c *client) ExecStep(ctx context.Context, ctn *pipeline.Container) error { // defer taking a snapshot of the step // // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Snapshot - defer func() { step.Snapshot(ctn, c.build, nil, nil, _step) }() + defer func() { step.Snapshot(ctx, ctn, c.build, nil, nil, _step) }() // run the runtime container err = c.Runtime.RunContainer(ctx, ctn, c.pipeline) @@ -177,7 +177,7 @@ func (c *client) DestroyStep(ctx context.Context, ctn *pipeline.Container) error // defer an upload of the step // // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Upload - defer func() { step.Upload(ctn, c.build, nil, nil, _step) }() + defer func() { step.Upload(ctx, ctn, c.build, nil, nil, _step) }() // inspect the runtime container err = c.Runtime.InspectContainer(ctx, ctn) diff --git a/go.mod b/go.mod index 7b69b3f1..69b98bc2 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,8 @@ require ( github.com/docker/docker v28.5.1+incompatible github.com/docker/go-units v0.5.0 github.com/gin-gonic/gin v1.11.0 - github.com/go-vela/sdk-go v0.27.2-0.20251023182037-f8a9d8e9a0a9 - github.com/go-vela/server v0.27.3-0.20251023181434-3790af6cb5db + github.com/go-vela/sdk-go v0.27.2-0.20251230154052-e1dae5f07d93 + github.com/go-vela/server v0.27.3-0.20251230152334-d014b0feb8db github.com/golang-jwt/jwt/v5 v5.3.0 github.com/google/go-cmp v0.7.0 github.com/joho/godotenv v1.5.1 @@ -98,7 +98,6 @@ require ( github.com/PuerkitoBio/purell v1.2.1 // indirect github.com/alicebob/miniredis/v2 v2.35.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/buildkite/yaml v0.0.0-20230306222819-0e4e032d4835 // indirect github.com/bytedance/sonic v1.14.0 // indirect github.com/bytedance/sonic/loader v0.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 3e5b2702..46e41c4f 100644 --- a/go.sum +++ b/go.sum @@ -34,8 +34,6 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/buildkite/yaml v0.0.0-20230306222819-0e4e032d4835 h1:Zfkih+Opdv9y5AOob+8iMsaMYnans+Ozrkb8wiPHbj0= -github.com/buildkite/yaml v0.0.0-20230306222819-0e4e032d4835/go.mod h1:AV5wtJnn1/CRaRGlJ8xspkMWfKXV0/pkJVgGleTIrfk= github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= @@ -123,10 +121,10 @@ github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHO github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/go-vela/sdk-go v0.27.2-0.20251023182037-f8a9d8e9a0a9 h1:jHgyONH8Mt91qkDtbLyN+F4RFoZSZboYZY3U5FoB8vw= -github.com/go-vela/sdk-go v0.27.2-0.20251023182037-f8a9d8e9a0a9/go.mod h1:wJF3oFa934D+7O+/7ruDz3h51FeePFZPP40zt1kg7Ac= -github.com/go-vela/server v0.27.3-0.20251023181434-3790af6cb5db h1:4/7TFAENWqGloUg2TAJNDKEj4b/t/f2iLKOLJuMUsXU= -github.com/go-vela/server v0.27.3-0.20251023181434-3790af6cb5db/go.mod h1:ziaGq1cWl+y6C+Vz+FxvuiqZnDtzRneqDqTuzTra8VE= +github.com/go-vela/sdk-go v0.27.2-0.20251230154052-e1dae5f07d93 h1:YzcZG/5nNGC8OylBkQkdDk47ikfWjHx1J01Gk8R32Uk= +github.com/go-vela/sdk-go v0.27.2-0.20251230154052-e1dae5f07d93/go.mod h1:hTkiVuPlurq/r/bAO0e74oDl0YeBfeHGbRI9H9MZVFA= +github.com/go-vela/server v0.27.3-0.20251230152334-d014b0feb8db h1:vVyCKnfsJhrXS3oT6cCxv4V4/M3plFAc/Fs/eAlUX4Q= +github.com/go-vela/server v0.27.3-0.20251230152334-d014b0feb8db/go.mod h1:8nFIwIrcWsfHzKYgMlj7rkN3pYdnSOVfcM7dJ2ci3b8= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= diff --git a/internal/build/snapshot.go b/internal/build/snapshot.go index f07a00aa..0b8155f6 100644 --- a/internal/build/snapshot.go +++ b/internal/build/snapshot.go @@ -3,6 +3,7 @@ package build import ( + "context" "strings" "time" @@ -15,7 +16,7 @@ import ( // Snapshot creates a moment in time record of the build // and attempts to upload it to the server. -func Snapshot(b *api.Build, c *vela.Client, e error, l *logrus.Entry) { +func Snapshot(ctx context.Context, b *api.Build, c *vela.Client, e error, l *logrus.Entry) { // check if the build is not in a canceled status if !strings.EqualFold(b.GetStatus(), constants.StatusCanceled) { // check if the error provided is empty @@ -42,7 +43,7 @@ func Snapshot(b *api.Build, c *vela.Client, e error, l *logrus.Entry) { // send API call to update the build // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#BuildService.Update - _, _, err := c.Build.Update(b) + _, _, err := c.Build.Update(ctx, b) if err != nil { l.Errorf("unable to upload build snapshot: %v", err) } diff --git a/internal/build/snapshot_test.go b/internal/build/snapshot_test.go index 9658f908..dfaaf15d 100644 --- a/internal/build/snapshot_test.go +++ b/internal/build/snapshot_test.go @@ -103,7 +103,7 @@ func TestBuild_Snapshot(t *testing.T) { // run test for _, test := range tests { t.Run(test.name, func(_ *testing.T) { - Snapshot(test.build, test.client, test.err, nil) + Snapshot(t.Context(), test.build, test.client, test.err, nil) }) } } diff --git a/internal/build/upload.go b/internal/build/upload.go index b20ecfc4..76cb10b9 100644 --- a/internal/build/upload.go +++ b/internal/build/upload.go @@ -3,6 +3,7 @@ package build import ( + "context" "strings" "time" @@ -15,7 +16,7 @@ import ( // Upload tracks the final state of the build // and attempts to upload it to the server. -func Upload(b *api.Build, c *vela.Client, e error, l *logrus.Entry) { +func Upload(ctx context.Context, b *api.Build, c *vela.Client, e error, l *logrus.Entry) { // handle the build based off the status provided switch b.GetStatus() { // build is in a canceled state @@ -70,7 +71,7 @@ func Upload(b *api.Build, c *vela.Client, e error, l *logrus.Entry) { // send API call to update the build // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#BuildService.Update - _, _, err := c.Build.Update(b) + _, _, err := c.Build.Update(ctx, b) if err != nil { l.Errorf("unable to upload final build state: %v", err) } diff --git a/internal/build/upload_test.go b/internal/build/upload_test.go index 41a2e929..63a911df 100644 --- a/internal/build/upload_test.go +++ b/internal/build/upload_test.go @@ -130,7 +130,7 @@ func TestBuild_Upload(t *testing.T) { // run test for _, test := range tests { t.Run(test.name, func(_ *testing.T) { - Upload(test.build, test.client, test.err, nil) + Upload(t.Context(), test.build, test.client, test.err, nil) }) } } diff --git a/internal/service/snapshot.go b/internal/service/snapshot.go index b4f52086..7883fae7 100644 --- a/internal/service/snapshot.go +++ b/internal/service/snapshot.go @@ -3,6 +3,7 @@ package service import ( + "context" "strings" "time" @@ -16,7 +17,7 @@ import ( // Snapshot creates a moment in time record of the // service and attempts to upload it to the server. -func Snapshot(ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logrus.Entry, s *api.Service) { +func Snapshot(ctx context.Context, ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logrus.Entry, s *api.Service) { // check if the build is not in a canceled status if !strings.EqualFold(s.GetStatus(), constants.StatusCanceled) { // check if the container is running in headless mode @@ -55,7 +56,7 @@ func Snapshot(ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logrus.E // send API call to update the service // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#SvcService.Update - _, _, err := c.Svc.Update(b.GetRepo().GetOrg(), b.GetRepo().GetName(), b.GetNumber(), s) + _, _, err := c.Svc.Update(ctx, b.GetRepo().GetOrg(), b.GetRepo().GetName(), b.GetNumber(), s) if err != nil { l.Errorf("unable to upload service snapshot: %v", err) } diff --git a/internal/service/snapshot_test.go b/internal/service/snapshot_test.go index 0dcecafc..292b8e00 100644 --- a/internal/service/snapshot_test.go +++ b/internal/service/snapshot_test.go @@ -135,7 +135,7 @@ func TestService_Snapshot(t *testing.T) { // run test for _, test := range tests { t.Run(test.name, func(_ *testing.T) { - Snapshot(test.container, test.build, test.client, nil, test.service) + Snapshot(t.Context(), test.container, test.build, test.client, nil, test.service) }) } } diff --git a/internal/service/upload.go b/internal/service/upload.go index cf43c436..55aafccb 100644 --- a/internal/service/upload.go +++ b/internal/service/upload.go @@ -3,6 +3,7 @@ package service import ( + "context" "time" "github.com/sirupsen/logrus" @@ -15,7 +16,7 @@ import ( // Upload tracks the final state of the service // and attempts to upload it to the server. -func Upload(ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logrus.Entry, s *api.Service) { +func Upload(ctx context.Context, ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logrus.Entry, s *api.Service) { // handle the service based off the status provided switch s.GetStatus() { // service is in a canceled state @@ -80,7 +81,7 @@ func Upload(ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logrus.Ent // send API call to update the service // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#SvcService.Update - _, _, err := c.Svc.Update(b.GetRepo().GetOrg(), b.GetRepo().GetName(), b.GetNumber(), s) + _, _, err := c.Svc.Update(ctx, b.GetRepo().GetOrg(), b.GetRepo().GetName(), b.GetNumber(), s) if err != nil { l.Errorf("unable to upload service snapshot: %v", err) } diff --git a/internal/service/upload_test.go b/internal/service/upload_test.go index c9d5011d..6b0fb95a 100644 --- a/internal/service/upload_test.go +++ b/internal/service/upload_test.go @@ -164,7 +164,7 @@ func TestService_Upload(t *testing.T) { // run test for _, test := range tests { t.Run(test.name, func(_ *testing.T) { - Upload(test.container, test.build, test.client, nil, test.service) + Upload(t.Context(), test.container, test.build, test.client, nil, test.service) }) } } diff --git a/internal/step/snapshot.go b/internal/step/snapshot.go index 4d67d390..5c5d085b 100644 --- a/internal/step/snapshot.go +++ b/internal/step/snapshot.go @@ -3,6 +3,7 @@ package step import ( + "context" "strings" "time" @@ -16,7 +17,7 @@ import ( // Snapshot creates a moment in time record of the // step and attempts to upload it to the server. -func Snapshot(ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logrus.Entry, s *api.Step) { +func Snapshot(ctx context.Context, ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logrus.Entry, s *api.Step) { // check if the build is not in a canceled status or error status logrus.Debugf("Snapshot s: %s %s", s.GetName(), s.GetStatus()) @@ -58,7 +59,7 @@ func Snapshot(ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logrus.E // send API call to update the step // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#StepService.Update - _, _, err := c.Step.Update(b.GetRepo().GetOrg(), b.GetRepo().GetName(), b.GetNumber(), s) + _, _, err := c.Step.Update(ctx, b.GetRepo().GetOrg(), b.GetRepo().GetName(), b.GetNumber(), s) if err != nil { l.Errorf("unable to upload step snapshot: %v", err) } @@ -67,7 +68,7 @@ func Snapshot(ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logrus.E // SnapshotInit creates a moment in time record of the // init step and attempts to upload it to the server. -func SnapshotInit(ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logrus.Entry, s *api.Step, lg *api.Log) { +func SnapshotInit(ctx context.Context, ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logrus.Entry, s *api.Step, lg *api.Log) { // check if the build is not in a canceled status if !strings.EqualFold(s.GetStatus(), constants.StatusCanceled) { // check if the container has an unsuccessful exit code @@ -99,7 +100,7 @@ func SnapshotInit(ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logr // send API call to update the step // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#StepService.Update - _, _, err := c.Step.Update(b.GetRepo().GetOrg(), b.GetRepo().GetName(), b.GetNumber(), s) + _, _, err := c.Step.Update(ctx, b.GetRepo().GetOrg(), b.GetRepo().GetName(), b.GetNumber(), s) if err != nil { l.Errorf("unable to upload step snapshot: %v", err) } @@ -109,7 +110,7 @@ func SnapshotInit(ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logr // send API call to update the logs for the step // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#LogService.UpdateStep - _, err = c.Log.UpdateStep(b.GetRepo().GetOrg(), b.GetRepo().GetName(), b.GetNumber(), s.GetNumber(), lg) + _, err = c.Log.UpdateStep(ctx, b.GetRepo().GetOrg(), b.GetRepo().GetName(), b.GetNumber(), s.GetNumber(), lg) if err != nil { l.Errorf("unable to upload step logs: %v", err) } diff --git a/internal/step/snapshot_test.go b/internal/step/snapshot_test.go index 9faa17a7..96b0cdac 100644 --- a/internal/step/snapshot_test.go +++ b/internal/step/snapshot_test.go @@ -134,7 +134,7 @@ func TestStep_Snapshot(t *testing.T) { // run test for _, test := range tests { t.Run(test.name, func(_ *testing.T) { - Snapshot(test.container, test.build, test.client, nil, test.step) + Snapshot(t.Context(), test.container, test.build, test.client, nil, test.step) }) } } @@ -261,7 +261,7 @@ func TestStep_SnapshotInit(t *testing.T) { // run test for _, test := range tests { t.Run(test.name, func(_ *testing.T) { - SnapshotInit(test.container, test.build, test.client, nil, test.step, test.log) + SnapshotInit(t.Context(), test.container, test.build, test.client, nil, test.step, test.log) }) } } diff --git a/internal/step/upload.go b/internal/step/upload.go index 5fe38454..74f30f6d 100644 --- a/internal/step/upload.go +++ b/internal/step/upload.go @@ -3,6 +3,7 @@ package step import ( + "context" "time" "github.com/sirupsen/logrus" @@ -15,7 +16,7 @@ import ( // Upload tracks the final state of the step // and attempts to upload it to the server. -func Upload(ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logrus.Entry, s *api.Step) { +func Upload(ctx context.Context, ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logrus.Entry, s *api.Step) { // handle the step based off the status provided switch s.GetStatus() { // step is in a canceled state @@ -80,7 +81,7 @@ func Upload(ctn *pipeline.Container, b *api.Build, c *vela.Client, l *logrus.Ent // send API call to update the step // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#StepService.Update - _, _, err := c.Step.Update(b.GetRepo().GetOrg(), b.GetRepo().GetName(), b.GetNumber(), s) + _, _, err := c.Step.Update(ctx, b.GetRepo().GetOrg(), b.GetRepo().GetName(), b.GetNumber(), s) if err != nil { l.Errorf("unable to upload final step state: %v", err) } diff --git a/internal/step/upload_test.go b/internal/step/upload_test.go index fc285592..0c2f907a 100644 --- a/internal/step/upload_test.go +++ b/internal/step/upload_test.go @@ -164,7 +164,7 @@ func TestStep_Upload(t *testing.T) { // run test for _, test := range tests { t.Run(test.name, func(_ *testing.T) { - Upload(test.container, test.build, test.client, nil, test.step) + Upload(t.Context(), test.container, test.build, test.client, nil, test.step) }) } } diff --git a/router/middleware/perm/perm.go b/router/middleware/perm/perm.go index d5bf5ec8..6bdf4cea 100644 --- a/router/middleware/perm/perm.go +++ b/router/middleware/perm/perm.go @@ -60,7 +60,7 @@ func MustServer() gin.HandlerFunc { vela.Authentication.SetTokenAuth(tkn) // validate the token with the configured vela server - resp, err := vela.Authentication.ValidateToken() + resp, err := vela.Authentication.ValidateToken(c.Request.Context()) if err != nil { msg := fmt.Sprintf("error validating token: %s", err) From bcdc2b4f7f7294c6ca0bd92dad146c049991ec0c Mon Sep 17 00:00:00 2001 From: ecrupper Date: Tue, 30 Dec 2025 09:52:44 -0600 Subject: [PATCH 35/63] restore some defaults --- docker-compose.yml | 2 +- executor/flags.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1436bc03..0c081b6c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,7 +51,7 @@ services: # https://go-vela.github.io/docs/administration/server/ server: container_name: server - image: server:local + image: target/vela-server:latest networks: - vela environment: diff --git a/executor/flags.go b/executor/flags.go index ffd70d90..c3d53cf1 100644 --- a/executor/flags.go +++ b/executor/flags.go @@ -40,7 +40,7 @@ var Flags = []cli.Flag{ Name: "executor.log_streaming_timeout", Usage: "maximum amount of time to wait for log streaming after build completes", Sources: cli.EnvVars("WORKER_LOG_STREAMING_TIMEOUT", "VELA_LOG_STREAMING_TIMEOUT", "LOG_STREAMING_TIMEOUT"), - Value: 1 * time.Minute, + Value: 5 * time.Minute, }, &cli.BoolFlag{ Name: "executor.enforce-trusted-repos", From 8a358d7e26cd1350487d1ab3de1d92ef24156a65 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Tue, 30 Dec 2025 10:08:26 -0600 Subject: [PATCH 36/63] linter --- cmd/vela-worker/operate.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/vela-worker/operate.go b/cmd/vela-worker/operate.go index 74f5272d..1d6b04c8 100644 --- a/cmd/vela-worker/operate.go +++ b/cmd/vela-worker/operate.go @@ -261,6 +261,8 @@ func (w *Worker) operate(ctx context.Context) error { // exec operator subprocess to poll and execute builds // (pass background context to avoid errors in one // executor+build inadvertently canceling other builds) + // + //nolint:contextcheck // see above err = w.exec(context.Background(), id, registryWorker) if err != nil { // log the error received from the executor From 36437faf5581a6cde3abbeba4dd73149d6b9d6d3 Mon Sep 17 00:00:00 2001 From: TimHuynh <163044036+Z003J76_tgt@users.noreply.github.com> Date: Tue, 3 Feb 2026 13:03:44 -0600 Subject: [PATCH 37/63] code change to support sts and remove database --- cmd/vela-worker/exec.go | 18 ---- cmd/vela-worker/operate.go | 53 ---------- cmd/vela-worker/run.go | 8 -- cmd/vela-worker/worker.go | 3 - executor/executor_test.go | 41 -------- executor/linux/build.go | 64 +----------- executor/linux/build_test.go | 104 -------------------- executor/linux/driver.go | 5 + executor/linux/linux.go | 2 - executor/linux/opts.go | 21 ---- executor/linux/outputs.go | 43 ++++---- executor/linux/stage.go | 2 +- executor/linux/stage_test.go | 69 ------------- executor/local/build.go | 2 +- executor/local/build_test.go | 103 ------------------- executor/local/local.go | 2 - executor/local/opts.go | 20 ---- executor/local/stage.go | 2 +- executor/local/stage_test.go | 18 ---- executor/setup.go | 21 ---- executor/setup_test.go | 54 ---------- internal/step/skip.go | 15 +-- internal/step/skip_test.go | 24 +---- router/middleware/executor/executor_test.go | 35 ------- 24 files changed, 32 insertions(+), 697 deletions(-) diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index 0c5afdc5..e9a04781 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -17,7 +17,6 @@ import ( "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" "github.com/go-vela/server/queue/models" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor" "github.com/go-vela/worker/runtime" "github.com/go-vela/worker/version" @@ -29,9 +28,6 @@ import ( //nolint:gocyclo,funlen // ignore cyclomatic complexity and function length func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error { var err error - - var execStorage storage.Storage - var _executor executor.Engine // setup the version @@ -166,14 +162,6 @@ func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error execOutputCtn := *w.Config.Executor.OutputCtn execOutputCtn.ID = fmt.Sprintf("outputs_%s", p.ID) - if w.Storage != nil { - execStorage = w.Storage - - logrus.Debugf("executor storage is available, setting up storage") - } else { - logrus.Debugf("executor storage is nil, skipping storage setup") - } - // create logger with extra metadata // // https://pkg.go.dev/github.com/sirupsen/logrus#WithFields @@ -268,12 +256,6 @@ func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error OutputCtn: &execOutputCtn, } - if execStorage != nil { - fmt.Printf("setting up executor storage\n") - - setup.Storage = execStorage - } - _executor, err = executor.New(setup) if err != nil { logger.Errorf("unable to setup executor: %v", err) diff --git a/cmd/vela-worker/operate.go b/cmd/vela-worker/operate.go index c931bf75..7b8eee48 100644 --- a/cmd/vela-worker/operate.go +++ b/cmd/vela-worker/operate.go @@ -12,7 +12,6 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/constants" "github.com/go-vela/server/queue" - "github.com/go-vela/server/storage" ) // operate is a helper function to initiate all @@ -78,58 +77,6 @@ func (w *Worker) operate(ctx context.Context) error { w.updateWorkerStatus(ctx, registryWorker, constants.WorkerStatusError) } - // getting storage creds - logrus.Trace("getting storage s3 creds..") - // fetching queue credentials using registration token - stCreds, _, err := w.VelaClient.Storage.GetInfo(ctx) - if err != nil { - logrus.Tracef("error getting storage creds: %v", err) - return err - } - - if stCreds.GetEnabled() { - logrus.Trace("storage enabled") - // if an address was given at start up, use that — else use what is returned from server - if len(w.Config.Storage.Endpoint) == 0 { - w.Config.Storage.Endpoint = stCreds.GetStorageAddress() - } - - // set access key in storage config - w.Config.Storage.AccessKey = stCreds.GetAccessKey() - - // set secret key in storage config - w.Config.Storage.SecretKey = stCreds.GetSecretKey() - - // set bucket name in storage config - w.Config.Storage.Bucket = stCreds.GetStorageBucket() - - // set storage enabled to true - w.Config.Storage.Enable = stCreds.GetEnabled() - - s, err := storage.New(w.Config.Storage) - if err != nil { - logrus.Error("storage setup failed") - // set to error as storage setup fails - w.updateWorkerStatus(ctx, registryWorker, constants.WorkerStatusError) - - return err - } - - w.Storage = s - logrus.WithFields(logrus.Fields{ - "driver": w.Config.Storage.Driver, - "bucket": w.Config.Storage.Bucket, - "endpoint": w.Config.Storage.Endpoint, - }).Debug("storage initialized") - } else { - logrus.Trace("storage not enabled") - // storage disabled; nothing to validate - w.Config.Storage.Enable = false - w.Storage = nil - - logrus.Debug("storage disabled: worker storage unset") - } - // spawn goroutine for phoning home executors.Go(func() error { // five second ticker for signal handling diff --git a/cmd/vela-worker/run.go b/cmd/vela-worker/run.go index 38d43fab..6d1e419d 100644 --- a/cmd/vela-worker/run.go +++ b/cmd/vela-worker/run.go @@ -17,7 +17,6 @@ import ( "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" "github.com/go-vela/server/queue" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor" "github.com/go-vela/worker/runtime" ) @@ -139,13 +138,6 @@ func run(ctx context.Context, c *cli.Command) error { Address: c.String("server.addr"), Secret: c.String("server.secret"), }, - Storage: &storage.Setup{ - Driver: c.String("storage.driver"), - Endpoint: c.String("storage.endpoint.name"), - AccessKey: c.String("storage.access.key"), - SecretKey: c.String("storage.secret.key"), - Bucket: c.String("storage.bucket.name"), - }, // Certificate configuration Certificate: &Certificate{ Cert: c.String("server.cert"), diff --git a/cmd/vela-worker/worker.go b/cmd/vela-worker/worker.go index d17acf10..ec6fcae4 100644 --- a/cmd/vela-worker/worker.go +++ b/cmd/vela-worker/worker.go @@ -10,7 +10,6 @@ import ( "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/queue" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor" "github.com/go-vela/worker/runtime" ) @@ -58,7 +57,6 @@ type ( Server *Server Certificate *Certificate TLSMinVersion string - Storage *storage.Setup } // Worker represents all configuration and @@ -74,6 +72,5 @@ type ( RunningBuilds []*api.Build QueueCheckedIn bool RunningBuildsMutex sync.Mutex - Storage storage.Storage } ) diff --git a/executor/executor_test.go b/executor/executor_test.go index 238f545c..9540a52f 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -16,7 +16,6 @@ import ( "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" "github.com/go-vela/server/mock/server" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor/linux" "github.com/go-vela/worker/executor/local" "github.com/go-vela/worker/runtime/docker" @@ -38,38 +37,6 @@ func TestExecutor_New(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } - _storageT := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _sT, err := storage.New(_storageT) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - - _storageF := &storage.Setup{ - Enable: false, - Driver: "", - Endpoint: "", - AccessKey: "", - SecretKey: "", - Bucket: "", - Region: "", - Secure: false, - } - - _sF, err := storage.New(_storageF) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - _linux, err := linux.New( linux.WithBuild(_build), linux.WithHostname("localhost"), @@ -78,7 +45,6 @@ func TestExecutor_New(t *testing.T) { linux.WithRuntime(_runtime), linux.WithVelaClient(_client), linux.WithVersion("v1.0.0"), - linux.WithStorage(_sT), ) if err != nil { t.Errorf("unable to create linux engine: %v", err) @@ -91,7 +57,6 @@ func TestExecutor_New(t *testing.T) { local.WithRuntime(_runtime), local.WithVelaClient(_client), local.WithVersion("v1.0.0"), - local.WithStorage(_sT), ) if err != nil { t.Errorf("unable to create local engine: %v", err) @@ -115,7 +80,6 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: _sF, }, want: nil, equal: reflect.DeepEqual, @@ -131,7 +95,6 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: _sT, }, want: _linux, equal: linux.Equal, @@ -146,7 +109,6 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: _sT, }, want: _local, equal: local.Equal, @@ -161,7 +123,6 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: _sT, }, want: nil, equal: reflect.DeepEqual, @@ -176,7 +137,6 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: _sT, }, want: nil, equal: reflect.DeepEqual, @@ -191,7 +151,6 @@ func TestExecutor_New(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: _sT, }, want: nil, equal: reflect.DeepEqual, diff --git a/executor/linux/build.go b/executor/linux/build.go index 89cca9f6..b6c3b5f1 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -521,7 +521,7 @@ func (c *client) ExecBuild(ctx context.Context) error { // check if the step should be skipped // // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Skip - skip, err := step.Skip(_step, c.build, c.build.GetStatus(), c.Storage) + skip, err := step.Skip(_step, c.build, c.build.GetStatus()) if err != nil { return fmt.Errorf("unable to plan step: %w", c.err) } @@ -530,64 +530,6 @@ func (c *client) ExecBuild(ctx context.Context) error { continue } - // Check if this step has artifacts and storage is disabled - //if !_step.Artifacts.Empty() && c.Storage == nil { - // c.Logger.Infof("skipping %s step: storage is disabled but artifacts is defined", _step.Name) - // - // //// Load step model - // //stepData, err := step.Load(_step, &c.steps) - // //if err != nil { - // // return fmt.Errorf("unable to load step: %w", err) - // //} - // // - // //// Load or create logs for this step - // ////stepLog, err := step.LoadLogs(_step, &c.stepLogs) - // ////if err != nil { - // //// return fmt.Errorf("unable to load step logs: %w", err) - // ////} - // // - // //// Ensure timestamps - // //now := time.Now().UTC().Unix() - // //if stepData.GetStarted() == 0 { - // // stepData.SetStarted(now) - // //} - // // - // //stepData.SetStatus(constants.StatusError) - // //stepData.SetExitCode(0) - // //stepData.SetFinished(now) - // - // // send API call to update the step - // // - // // https://pkg.go.dev/github.com/go-vela/sdk-go/vela#StepService.Update - // //_tsstep, _, err := c.Vela.Step.Update(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), stepData) - // //if err != nil { - // // return err - // //} - // // - // //// send API call to capture the step log - // //// - // //// https://pkg.go.dev/github.com/go-vela/sdk-go/vela#LogService.GetStep - // //_log, _, err := c.Vela.Log.GetStep(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), _tsstep.GetNumber()) - // //if err != nil { - // // return err - // //} - // //_log.AppendData([]byte("Storage is disabled, contact Vela Admins\n")) - // // - // //// add a step log to a map - // //c.stepLogs.Store(_step.ID, _log) - // //stepLog.AppendData([]byte("Storage is disabled, contact Vela Admins\n")) - // //stepLog.SetData([]byte("Storage is disabled, contact Vela Admins\n")) - // //// Upload logs so UI can display the message - // //if _, err := c.Vela.Log. - // // UpdateStep(c.build.GetRepo().GetOrg(), c.build.GetRepo().GetName(), c.build.GetNumber(), *stepData.Number, stepLog); err != nil { - // // c.Logger.Errorf("unable to upload skipped step logs: %v", err) - // //} - // // Upload step status - // //step.Upload(_step, c.build, c.Vela, c.Logger, stepData) - // - // continue - //} - // add netrc to secrets for masking in logs sec := &pipeline.StepSecret{ Target: "VELA_NETRC_PASSWORD", @@ -615,9 +557,6 @@ func (c *client) ExecBuild(ctx context.Context) error { _step.Secrets = append(_step.Secrets, sec) } - // logic for polling files only if the artifacts step is present - // iterate through the steps in the build - // TODO: API to return if storage is enabled //if c.Storage == nil && _step.Artifacts.Empty() || c.Storage == nil && !_step.Artifacts.Empty() { // c.Logger.Infof("storage disabled, skipping artifacts for %s step", _step.Name) @@ -631,7 +570,6 @@ func (c *client) ExecBuild(ctx context.Context) error { c.Logger.Errorf("unable to poll files for artifacts: %v", err) } } - //} // perform any substitution on dynamic variables err = _step.Substitute() diff --git a/executor/linux/build_test.go b/executor/linux/build_test.go index 41acd481..ad97dc35 100644 --- a/executor/linux/build_test.go +++ b/executor/linux/build_test.go @@ -21,7 +21,6 @@ import ( "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" "github.com/go-vela/server/mock/server" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime" "github.com/go-vela/worker/runtime/docker" @@ -186,30 +185,12 @@ func TestLinux_CreateBuild(t *testing.T) { } } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - _engine, err := New( WithLogger(logger), WithBuild(test.build), WithPipeline(_pipeline), WithRuntime(_runtime), - WithVelaClient(_client), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -280,22 +261,6 @@ func TestLinux_PlanBuild(t *testing.T) { t.Errorf("unable to create Vela API client: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - tests := []struct { name string failure bool @@ -419,7 +384,6 @@ func TestLinux_PlanBuild(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithVelaClient(_client), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -499,22 +463,6 @@ func TestLinux_AssembleBuild(t *testing.T) { streamRequests, done := message.MockStreamRequestsWithCancel(context.Background()) defer done() - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - tests := []struct { name string failure bool @@ -736,7 +684,6 @@ func TestLinux_AssembleBuild(t *testing.T) { WithVelaClient(_client), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -837,22 +784,6 @@ func TestLinux_ExecBuild(t *testing.T) { t.Errorf("unable to create Vela API client: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - tests := []struct { name string failure bool @@ -1012,7 +943,6 @@ func TestLinux_ExecBuild(t *testing.T) { WithVelaClient(_client), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -1162,22 +1092,6 @@ func TestLinux_StreamBuild(t *testing.T) { t.Errorf("unable to create Vela API client: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - type planFuncType = func(context.Context, *pipeline.Container) error // planNothing is a planFuncType that does nothing @@ -1684,7 +1598,6 @@ func TestLinux_StreamBuild(t *testing.T) { WithLogStreamingTimeout(1*time.Second), WithVelaClient(_client), withStreamRequests(streamRequests), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -1811,22 +1724,6 @@ func TestLinux_DestroyBuild(t *testing.T) { t.Errorf("unable to create Vela API client: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - tests := []struct { name string failure bool @@ -1993,7 +1890,6 @@ func TestLinux_DestroyBuild(t *testing.T) { WithRuntime(_runtime), WithVelaClient(_client), WithOutputCtn(testOutputsCtn()), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) diff --git a/executor/linux/driver.go b/executor/linux/driver.go index 5ef1dd59..ae4821ea 100644 --- a/executor/linux/driver.go +++ b/executor/linux/driver.go @@ -8,3 +8,8 @@ import "github.com/go-vela/server/constants" func (c *client) Driver() string { return constants.DriverLinux } + +// StorageDriver outputs the configured storage driver. +func (c *client) StorageDriver() string { + return constants.DriverMinio +} diff --git a/executor/linux/linux.go b/executor/linux/linux.go index c9e8bd48..70502d85 100644 --- a/executor/linux/linux.go +++ b/executor/linux/linux.go @@ -13,7 +13,6 @@ import ( "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime" ) @@ -30,7 +29,6 @@ type ( Hostname string Version string OutputCtn *pipeline.Container - Storage storage.Storage // clients for build actions secret *secretSvc diff --git a/executor/linux/opts.go b/executor/linux/opts.go index 33d982c8..5c394bae 100644 --- a/executor/linux/opts.go +++ b/executor/linux/opts.go @@ -11,7 +11,6 @@ import ( "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime" ) @@ -210,23 +209,3 @@ func withStreamRequests(s chan message.StreamRequest) Opt { return nil } } - -// WithStorage sets the storage in the executor client for Linux. -func WithStorage(s storage.Storage) Opt { - return func(c *client) error { - c.Logger.Trace("configuring storage in linux executor client") - - // check if the storage provided is empty - if s == nil { - return fmt.Errorf("empty storage setup provided") - } - - c.Storage = s - - if c.Storage == nil { - return fmt.Errorf("empty storage setup in linux executor client") - } - - return nil - } -} diff --git a/executor/linux/outputs.go b/executor/linux/outputs.go index 172149ad..aff87fd8 100644 --- a/executor/linux/outputs.go +++ b/executor/linux/outputs.go @@ -12,6 +12,8 @@ import ( "strconv" envparse "github.com/hashicorp/go-envparse" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/sirupsen/logrus" api "github.com/go-vela/server/api/types" @@ -220,6 +222,24 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, file // https://pkg.go.dev/github.com/sirupsen/logrus#Entry.WithField logger := o.client.Logger.WithField("test-outputs", ctn.Name) + creds, res, err := o.client.Vela.Storage.GetSTSCreds(ctx, b.GetRepo().GetOrg(), b.GetRepo().GetName(), + b.GetNumber()) + if err != nil { + return fmt.Errorf("unable to get sts storage creds %w with response code %d", err, res.StatusCode) + } + logrus.Debugf("STS endpoint at: %s and res code %d", creds.Endpoint, res.StatusCode) + // 3) build a storage engine using the temp creds (or build a minio client directly) + stsStorageClient, err := minio.New(creds.Endpoint, + &minio.Options{ + Creds: credentials.NewStaticV4(creds.AccessKey, creds.SecretKey, creds.SessionToken), + }) + if err != nil { + return fmt.Errorf("unable to create sts storage client %w with response code %d", err, res.StatusCode) + } + if stsStorageClient == nil { + return fmt.Errorf("sts storage client is nil") + } + // grab file paths from the container filesPath, err := o.client.Runtime.PollFileNames(ctx, ctn, fileList) if err != nil { @@ -249,31 +269,10 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, file fileName) // upload file to storage - err = o.client.Storage.UploadObject(ctx, &api.Object{ - ObjectName: objectName, - Bucket: api.Bucket{BucketName: o.client.Storage.GetBucket(ctx)}, - FilePath: filePath, - }, reader, size) + _, err = stsStorageClient.PutObject(ctx, creds.Bucket, objectName, reader, size, minio.PutObjectOptions{}) if err != nil { return fmt.Errorf("unable to upload object %s: %w", fileName, err) } - - presignURL, err := o.client.Storage.PresignedGetObject(ctx, &api.Object{ - ObjectName: objectName, - Bucket: api.Bucket{BucketName: o.client.Storage.GetBucket(ctx)}, - FilePath: filePath, - }) - if err != nil { - return fmt.Errorf("unable to generate presign URL for %s: %w", fileName, err) - } - - // create artifact record in database after successful upload - err = o.client.CreateArtifact(ctx, fileName, presignURL, size) - if err != nil { - logger.Errorf("unable to create artifact record for %s: %v", fileName, err) - // don't return error here to avoid blocking the upload process - } } - return nil } diff --git a/executor/linux/stage.go b/executor/linux/stage.go index 899afcd9..9db22fa4 100644 --- a/executor/linux/stage.go +++ b/executor/linux/stage.go @@ -132,7 +132,7 @@ func (c *client) ExecStage(ctx context.Context, s *pipeline.Stage, m *sync.Map) // check if the step should be skipped // // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Skip - skip, err := step.Skip(_step, c.build, useStatus, c.Storage) + skip, err := step.Skip(_step, c.build, useStatus) if err != nil { return fmt.Errorf("unable to plan step: %w", c.err) } diff --git a/executor/linux/stage_test.go b/executor/linux/stage_test.go index 58664b33..eaa3bae5 100644 --- a/executor/linux/stage_test.go +++ b/executor/linux/stage_test.go @@ -16,7 +16,6 @@ import ( "github.com/go-vela/server/compiler/native" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/mock/server" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime" "github.com/go-vela/worker/runtime/docker" @@ -73,22 +72,6 @@ func TestLinux_CreateStage(t *testing.T) { t.Errorf("unable to create kubernetes runtime engine: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - // setup tests tests := []struct { name string @@ -194,7 +177,6 @@ func TestLinux_CreateStage(t *testing.T) { WithPipeline(_pipeline), WithRuntime(test.runtime), WithVelaClient(_client), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -248,22 +230,6 @@ func TestLinux_PlanStage(t *testing.T) { t.Errorf("unable to create kubernetes runtime engine: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - dockerTestMap := new(sync.Map) dockerTestMap.Store("foo", make(chan error, 1)) @@ -438,7 +404,6 @@ func TestLinux_PlanStage(t *testing.T) { WithPipeline(new(pipeline.Build)), WithRuntime(test.runtime), WithVelaClient(_client), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -489,22 +454,6 @@ func TestLinux_ExecStage(t *testing.T) { streamRequests, done := message.MockStreamRequestsWithCancel(context.Background()) defer done() - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - // setup tests tests := []struct { name string @@ -644,7 +593,6 @@ func TestLinux_ExecStage(t *testing.T) { WithVelaClient(_client), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) @@ -690,22 +638,6 @@ func TestLinux_DestroyStage(t *testing.T) { t.Errorf("unable to create kubernetes runtime engine: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - // setup tests tests := []struct { name string @@ -761,7 +693,6 @@ func TestLinux_DestroyStage(t *testing.T) { WithPipeline(new(pipeline.Build)), WithRuntime(test.runtime), WithVelaClient(_client), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create %s executor engine: %v", test.name, err) diff --git a/executor/local/build.go b/executor/local/build.go index d6c65a3c..d5fcdcb3 100644 --- a/executor/local/build.go +++ b/executor/local/build.go @@ -289,7 +289,7 @@ func (c *client) ExecBuild(ctx context.Context) error { // check if the step should be skipped // // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Skip - skip, err := step.Skip(_step, c.build, c.build.GetStatus(), c.Storage) + skip, err := step.Skip(_step, c.build, c.build.GetStatus()) if err != nil { return fmt.Errorf("unable to plan step: %w", c.err) } diff --git a/executor/local/build_test.go b/executor/local/build_test.go index 94dba3dc..8687f122 100644 --- a/executor/local/build_test.go +++ b/executor/local/build_test.go @@ -11,7 +11,6 @@ import ( "github.com/go-vela/server/compiler/native" "github.com/go-vela/server/compiler/types/pipeline" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime/docker" ) @@ -39,22 +38,6 @@ func TestLocal_CreateBuild(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - tests := []struct { name string failure bool @@ -98,7 +81,6 @@ func TestLocal_CreateBuild(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -144,22 +126,6 @@ func TestLocal_PlanBuild(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - tests := []struct { name string failure bool @@ -203,7 +169,6 @@ func TestLocal_PlanBuild(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -255,22 +220,6 @@ func TestLocal_AssembleBuild(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - streamRequests, done := message.MockStreamRequestsWithCancel(context.Background()) defer done() @@ -348,7 +297,6 @@ func TestLocal_AssembleBuild(t *testing.T) { WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -400,22 +348,6 @@ func TestLocal_ExecBuild(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - streamRequests, done := message.MockStreamRequestsWithCancel(context.Background()) defer done() @@ -478,7 +410,6 @@ func TestLocal_ExecBuild(t *testing.T) { WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -530,22 +461,6 @@ func TestLocal_StreamBuild(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - type planFuncType = func(context.Context, *pipeline.Container) error // planNothing is a planFuncType that does nothing @@ -700,7 +615,6 @@ func TestLocal_StreamBuild(t *testing.T) { WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), withStreamRequests(streamRequests), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -772,22 +686,6 @@ func TestLocal_DestroyBuild(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - tests := []struct { name string failure bool @@ -846,7 +744,6 @@ func TestLocal_DestroyBuild(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) diff --git a/executor/local/local.go b/executor/local/local.go index ee8908c8..33f88a55 100644 --- a/executor/local/local.go +++ b/executor/local/local.go @@ -11,7 +11,6 @@ import ( "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime" ) @@ -24,7 +23,6 @@ type ( Hostname string Version string OutputCtn *pipeline.Container - Storage storage.Storage // private fields init *pipeline.Container diff --git a/executor/local/opts.go b/executor/local/opts.go index fb012db8..61660ed7 100644 --- a/executor/local/opts.go +++ b/executor/local/opts.go @@ -9,7 +9,6 @@ import ( "github.com/go-vela/sdk-go/vela" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime" ) @@ -141,22 +140,3 @@ func withStreamRequests(s chan message.StreamRequest) Opt { return nil } } - -// WithStorage sets the storage in the executor client for Linux. -func WithStorage(s storage.Storage) Opt { - return func(c *client) error { - // check if the storage provided is empty - if s == nil { - return fmt.Errorf("empty storage setup provided") - } - - // set the storage in the client - c.Storage = s - - if c.Storage == nil { - return fmt.Errorf("empty storage setup: %v", s) - } - - return nil - } -} diff --git a/executor/local/stage.go b/executor/local/stage.go index a050a4f7..05564596 100644 --- a/executor/local/stage.go +++ b/executor/local/stage.go @@ -109,7 +109,7 @@ func (c *client) ExecStage(ctx context.Context, s *pipeline.Stage, m *sync.Map) // check if the step should be skipped // // https://pkg.go.dev/github.com/go-vela/worker/internal/step#Skip - skip, err := step.Skip(_step, c.build, useStatus, c.Storage) + skip, err := step.Skip(_step, c.build, useStatus) if err != nil { return fmt.Errorf("unable to plan step: %w", c.err) } diff --git a/executor/local/stage_test.go b/executor/local/stage_test.go index d63b6c6e..2c880b7c 100644 --- a/executor/local/stage_test.go +++ b/executor/local/stage_test.go @@ -12,7 +12,6 @@ import ( "github.com/go-vela/server/compiler/native" "github.com/go-vela/server/compiler/types/pipeline" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/internal/message" "github.com/go-vela/worker/runtime/docker" ) @@ -52,22 +51,6 @@ func TestLocal_CreateStage(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - // setup tests tests := []struct { name string @@ -120,7 +103,6 @@ func TestLocal_CreateStage(t *testing.T) { WithPipeline(_pipeline), WithRuntime(_runtime), WithOutputCtn(testOutputsCtn()), - WithStorage(_s), ) if err != nil { t.Errorf("unable to create executor engine: %v", err) diff --git a/executor/setup.go b/executor/setup.go index 10f37f24..cda031dd 100644 --- a/executor/setup.go +++ b/executor/setup.go @@ -13,7 +13,6 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor/linux" "github.com/go-vela/worker/executor/local" "github.com/go-vela/worker/runtime" @@ -61,8 +60,6 @@ type Setup struct { Pipeline *pipeline.Build // id token request token for the build RequestToken string - // storage client for interacting with storage resources - Storage storage.Storage } // Darwin creates and returns a Vela engine capable of @@ -93,14 +90,6 @@ func (s *Setup) Linux() (Engine, error) { linux.WithLogger(s.Logger), linux.WithOutputCtn(s.OutputCtn), } - - // Conditionally add storage option - if s.Storage != nil { - fmt.Printf("Adding storage to linux executor\n") - - opts = append(opts, linux.WithStorage(s.Storage)) - } - // create new Linux executor engine // // https://pkg.go.dev/github.com/go-vela/worker/executor/linux#New @@ -123,11 +112,6 @@ func (s *Setup) Local() (Engine, error) { local.WithOutputCtn(s.OutputCtn), } - // Conditionally add storage option - if s.Storage != nil { - opts = append(opts, local.WithStorage(s.Storage)) - } - // create new Local executor engine // // https://pkg.go.dev/github.com/go-vela/worker/executor/local#New @@ -189,11 +173,6 @@ func (s *Setup) Validate() error { return fmt.Errorf("no Vela user provided in setup") } - // If storage is provided, ensure it's enabled - if s.Storage != nil && !s.Storage.StorageEnable() { - return fmt.Errorf("storage client provided but not enabled in setup") - } - // setup is valid return nil } diff --git a/executor/setup_test.go b/executor/setup_test.go index 87999425..e89700a7 100644 --- a/executor/setup_test.go +++ b/executor/setup_test.go @@ -14,7 +14,6 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/constants" "github.com/go-vela/server/mock/server" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor/linux" "github.com/go-vela/worker/executor/local" "github.com/go-vela/worker/runtime/docker" @@ -70,22 +69,6 @@ func TestExecutor_Setup_Linux(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - want, err := linux.New( linux.WithBuild(_build), linux.WithMaxLogSize(2097152), @@ -95,7 +78,6 @@ func TestExecutor_Setup_Linux(t *testing.T) { linux.WithRuntime(_runtime), linux.WithVelaClient(_client), linux.WithVersion("v1.0.0"), - linux.WithStorage(_s), ) if err != nil { t.Errorf("unable to create linux engine: %v", err) @@ -110,7 +92,6 @@ func TestExecutor_Setup_Linux(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: _s, } // run test @@ -142,22 +123,6 @@ func TestExecutor_Setup_Local(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - want, err := local.New( local.WithBuild(_build), local.WithHostname("localhost"), @@ -165,7 +130,6 @@ func TestExecutor_Setup_Local(t *testing.T) { local.WithRuntime(_runtime), local.WithVelaClient(_client), local.WithVersion("v1.0.0"), - local.WithStorage(_s), ) if err != nil { t.Errorf("unable to create local engine: %v", err) @@ -179,7 +143,6 @@ func TestExecutor_Setup_Local(t *testing.T) { Pipeline: _pipeline, Runtime: _runtime, Version: "v1.0.0", - Storage: _s, } // run test @@ -245,22 +208,6 @@ func TestExecutor_Setup_Validate(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - _emptyOwnerBuild := new(api.Build) _emptyOwnerBuild.SetRepo(new(api.Repo)) @@ -281,7 +228,6 @@ func TestExecutor_Setup_Validate(t *testing.T) { MaxLogSize: 2097152, Pipeline: _pipeline, Runtime: _runtime, - Storage: _s, }, failure: false, }, diff --git a/internal/step/skip.go b/internal/step/skip.go index 427c8123..b6f2ded7 100644 --- a/internal/step/skip.go +++ b/internal/step/skip.go @@ -8,30 +8,17 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" - "github.com/go-vela/server/storage" ) // Skip creates the ruledata from the build and repository // information and returns true if the data does not match // the ruleset for the given container. -func Skip(c *pipeline.Container, b *api.Build, status string, storage storage.Storage) (bool, error) { +func Skip(c *pipeline.Container, b *api.Build, status string) (bool, error) { // check if the container provided is empty if c == nil { return true, nil } - if !c.Artifacts.Empty() { - if storage == nil { - return true, nil - } - - if !storage.StorageEnable() { - return true, nil - } - - return false, nil - } - event := b.GetEvent() action := b.GetEventAction() diff --git a/internal/step/skip_test.go b/internal/step/skip_test.go index 6781972d..e23eca2f 100644 --- a/internal/step/skip_test.go +++ b/internal/step/skip_test.go @@ -9,7 +9,6 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" - "github.com/go-vela/server/storage" ) func TestStep_Skip(t *testing.T) { @@ -212,75 +211,54 @@ func TestStep_Skip(t *testing.T) { Name: constants.InitName, Number: 1, Pull: "always", - Artifacts: pipeline.Artifacts{ - Paths: []string{"foo.xml", "bar.xml", "foo.txt", "bar.txt"}, - }, - } - - s := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "minioadmin", - SecretKey: "minioadmin", - Bucket: "vela", } - _storage, _ := storage.New(s) tests := []struct { name string build *api.Build container *pipeline.Container - storage *storage.Storage want bool }{ { name: "build", build: _build, container: _container, - storage: &_storage, want: false, }, { name: "comment", build: _comment, container: _container, - storage: &_storage, want: false, }, { name: "deploy", build: _deploy, container: _container, - storage: &_storage, want: false, }, { name: "deployFromTag", build: _deployFromTag, container: _container, - storage: &_storage, want: false, }, { name: "schedule", build: _schedule, container: _container, - storage: &_storage, want: false, }, { name: "tag", build: _tag, container: _container, - storage: &_storage, want: false, }, { name: "skip nil", build: nil, container: nil, - storage: &_storage, want: true, }, } @@ -288,7 +266,7 @@ func TestStep_Skip(t *testing.T) { // run test for _, test := range tests { t.Run(test.name, func(t *testing.T) { - got, err := Skip(test.container, test.build, test.build.GetStatus(), *test.storage) + got, err := Skip(test.container, test.build, test.build.GetStatus()) if err != nil { t.Errorf("Skip returned error: %s", err) } diff --git a/router/middleware/executor/executor_test.go b/router/middleware/executor/executor_test.go index 09fbc9e1..ced87338 100644 --- a/router/middleware/executor/executor_test.go +++ b/router/middleware/executor/executor_test.go @@ -14,7 +14,6 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/constants" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor" "github.com/go-vela/worker/runtime/docker" ) @@ -34,22 +33,6 @@ func TestExecutor_Retrieve(t *testing.T) { _build := new(api.Build) _build.SetRepo(_repo) - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - want, err := executor.New(&executor.Setup{ Driver: constants.DriverLinux, MaxLogSize: 2097152, @@ -57,7 +40,6 @@ func TestExecutor_Retrieve(t *testing.T) { Runtime: _runtime, Build: _build, Pipeline: new(pipeline.Build), - Storage: _s, }) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -90,22 +72,6 @@ func TestExecutor_Establish(t *testing.T) { _build := new(api.Build) _build.SetRepo(_repo) - _storage := &storage.Setup{ - Enable: true, - Driver: "minio", - Endpoint: "http://localhost:9000", - AccessKey: "ad", - SecretKey: "asd", - Bucket: "vela", - Region: "", - Secure: false, - } - - _s, err := storage.New(_storage) - if err != nil { - t.Errorf("unable to create storage engine: %v", err) - } - want, err := executor.New(&executor.Setup{ Driver: constants.DriverLinux, MaxLogSize: 2097152, @@ -113,7 +79,6 @@ func TestExecutor_Establish(t *testing.T) { Runtime: _runtime, Build: _build, Pipeline: new(pipeline.Build), - Storage: _s, }) if err != nil { t.Errorf("unable to create executor engine: %v", err) From 57b4be096dabea437cdbb68bbcfbc1a0971957fd Mon Sep 17 00:00:00 2001 From: TimHuynh <163044036+Z003J76_tgt@users.noreply.github.com> Date: Tue, 3 Feb 2026 13:07:49 -0600 Subject: [PATCH 38/63] point gomod to feat_artifacts branches --- go.mod | 6 +++--- go.sum | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ddb71ef8..7b92a712 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,12 @@ require ( github.com/docker/docker v28.5.1+incompatible github.com/docker/go-units v0.5.0 github.com/gin-gonic/gin v1.11.0 - github.com/go-vela/sdk-go v0.27.2-0.20251230154052-e1dae5f07d93 - github.com/go-vela/server v0.27.3-0.20260107161845-56fd3db7dbff + github.com/go-vela/sdk-go v0.27.2-0.20260203190443-c7ae4e25a718 + github.com/go-vela/server v0.27.3-0.20260203190253-90deb1b0fdd7 github.com/golang-jwt/jwt/v5 v5.3.0 github.com/google/go-cmp v0.7.0 github.com/joho/godotenv v1.5.1 + github.com/minio/minio-go/v7 v7.0.89 github.com/opencontainers/image-spec v1.1.1 github.com/prometheus/client_golang v1.23.2 github.com/sirupsen/logrus v1.9.3 @@ -60,7 +61,6 @@ require ( github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/minio/crc64nvme v1.0.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.89 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/quic-go/qpack v0.6.0 // indirect diff --git a/go.sum b/go.sum index 15cc07c1..001a2222 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,11 @@ github.com/go-playground/validator/v10 v10.29.0 h1:lQlF5VNJWNlRbRZNeOIkWElR+1LL/ github.com/go-playground/validator/v10 v10.29.0/go.mod h1:D6QxqeMlgIPuT02L66f2ccrZ7AGgHkzKmmTMZhk/Kc4= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-vela/sdk-go v0.27.2-0.20251230154052-e1dae5f07d93/go.mod h1:hTkiVuPlurq/r/bAO0e74oDl0YeBfeHGbRI9H9MZVFA= +github.com/go-vela/sdk-go v0.27.2-0.20260203190443-c7ae4e25a718 h1:sFPUqpHVbBC9tviYakmhEtpSgUvbyd2xrAotme84/eE= +github.com/go-vela/sdk-go v0.27.2-0.20260203190443-c7ae4e25a718/go.mod h1:OXizg0doBUj9R4kKMCIx3KNoNxI8hyByjKTlLK86QRI= +github.com/go-vela/server v0.27.3-0.20260203190253-90deb1b0fdd7 h1:iXEnIUxBYWxnKqQfDtJJh3vHYod7KYzKVXDGsP1o37M= +github.com/go-vela/server v0.27.3-0.20260203190253-90deb1b0fdd7/go.mod h1:L83ces3y+5yNaPT5v35zPAzN5le8OROEYoTjldwgT88= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.0 h1:EmkZ9RIsX+Uq4DYFowegAuJo8+xdX3T/2dwNPXbxEYE= From 1a542f68e052506cfeb886de971c043a25c6ecef Mon Sep 17 00:00:00 2001 From: TimHuynh <163044036+Z003J76_tgt@users.noreply.github.com> Date: Tue, 3 Feb 2026 13:19:30 -0600 Subject: [PATCH 39/63] remove artfiact db --- executor/linux/artifact.go | 58 -------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 executor/linux/artifact.go diff --git a/executor/linux/artifact.go b/executor/linux/artifact.go deleted file mode 100644 index 088c081d..00000000 --- a/executor/linux/artifact.go +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package linux - -import ( - "context" - "fmt" - "path/filepath" - "strconv" - "time" - - api "github.com/go-vela/server/api/types" -) - -// CreateArtifact creates an artifact record in the database -// after a file has been successfully uploaded to storage. -func (c *client) CreateArtifact(ctx context.Context, fileName, presignURL string, size int64) error { - // extract file extension and type information - fileExt := filepath.Ext(fileName) - - // create object path matching the storage upload format - objectPath := fmt.Sprintf("%s/%s/%s/%s", - c.build.GetRepo().GetOrg(), - c.build.GetRepo().GetName(), - strconv.FormatInt(c.build.GetNumber(), 10), - fileName) - - // create timestamp for record creation - createdAt := time.Now().Unix() - - // build artifact record - artifact := &api.Artifact{ - BuildID: c.build.ID, - FileName: &fileName, - ObjectPath: &objectPath, - FileSize: &size, - FileType: &fileExt, - PresignedURL: &presignURL, - CreatedAt: &createdAt, - } - - // create artifact record in database - a, resp, err := c.Vela.Artifact.Update( - ctx, - c.build.GetRepo().GetOrg(), - c.build.GetRepo().GetName(), - c.build.GetNumber(), - artifact, - ) - if err != nil { - return fmt.Errorf("failed to create artifact record: build=%d, status=%d, error=%w", - c.build.GetNumber(), resp.StatusCode, err) - } - - c.Logger.Debugf("created artifact record: id=%d, file=%s", a.GetID(), fileName) - - return nil -} From 5a7acb67677517ab4ab26c851a9c0808e93ca0d2 Mon Sep 17 00:00:00 2001 From: TimHuynh <163044036+Z003J76_tgt@users.noreply.github.com> Date: Fri, 6 Feb 2026 15:03:53 -0600 Subject: [PATCH 40/63] use build service getSTS --- executor/linux/build.go | 21 +++++++-------------- executor/linux/outputs.go | 13 ++++++++----- go.mod | 4 ++-- go.sum | 9 ++++----- 4 files changed, 21 insertions(+), 26 deletions(-) diff --git a/executor/linux/build.go b/executor/linux/build.go index b6c3b5f1..0ab19aab 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -557,20 +557,6 @@ func (c *client) ExecBuild(ctx context.Context) error { _step.Secrets = append(_step.Secrets, sec) } - // TODO: API to return if storage is enabled - //if c.Storage == nil && _step.Artifacts.Empty() || c.Storage == nil && !_step.Artifacts.Empty() { - // c.Logger.Infof("storage disabled, skipping artifacts for %s step", _step.Name) - // // skip if no storage client - // // but artifacts is defined in step - // continue - //} else if !_step.Artifacts.Empty() && c.Storage != nil { - if len(_step.Artifacts.Paths) != 0 { - err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.Artifacts.Paths, c.build) - if err != nil { - c.Logger.Errorf("unable to poll files for artifacts: %v", err) - } - } - // perform any substitution on dynamic variables err = _step.Substitute() if err != nil { @@ -589,6 +575,13 @@ func (c *client) ExecBuild(ctx context.Context) error { if c.err != nil { return fmt.Errorf("unable to execute step: %w", c.err) } + + if len(_step.Artifacts.Paths) != 0 { + err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.Artifacts.Paths, c.build) + if err != nil { + c.Logger.Errorf("unable to poll files for artifacts: %v", err) + } + } } // create an error group with the context for each stage diff --git a/executor/linux/outputs.go b/executor/linux/outputs.go index aff87fd8..0c12c6df 100644 --- a/executor/linux/outputs.go +++ b/executor/linux/outputs.go @@ -220,18 +220,21 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, file // update engine logger with outputs metadata // // https://pkg.go.dev/github.com/sirupsen/logrus#Entry.WithField - logger := o.client.Logger.WithField("test-outputs", ctn.Name) + logger := o.client.Logger.WithField("artifact-outputs", ctn.Name) - creds, res, err := o.client.Vela.Storage.GetSTSCreds(ctx, b.GetRepo().GetOrg(), b.GetRepo().GetName(), + creds, res, err := o.client.Vela.Build.GetSTSCreds(ctx, b.GetRepo().GetOrg(), b.GetRepo().GetName(), b.GetNumber()) if err != nil { return fmt.Errorf("unable to get sts storage creds %w with response code %d", err, res.StatusCode) } - logrus.Debugf("STS endpoint at: %s and res code %d", creds.Endpoint, res.StatusCode) - // 3) build a storage engine using the temp creds (or build a minio client directly) + stsStorageClient, err := minio.New(creds.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(creds.AccessKey, creds.SecretKey, creds.SessionToken), + Creds: credentials.NewStaticV4( + creds.AccessKey, + creds.SecretKey, + creds.SessionToken), + Secure: creds.Secure, }) if err != nil { return fmt.Errorf("unable to create sts storage client %w with response code %d", err, res.StatusCode) diff --git a/go.mod b/go.mod index 7b92a712..a1c95a39 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,8 @@ require ( github.com/docker/docker v28.5.1+incompatible github.com/docker/go-units v0.5.0 github.com/gin-gonic/gin v1.11.0 - github.com/go-vela/sdk-go v0.27.2-0.20260203190443-c7ae4e25a718 - github.com/go-vela/server v0.27.3-0.20260203190253-90deb1b0fdd7 + github.com/go-vela/sdk-go v0.27.2-0.20260206210222-679cb21b266d + github.com/go-vela/server v0.27.3-0.20260206204023-9a8f2f0e09c3 github.com/golang-jwt/jwt/v5 v5.3.0 github.com/google/go-cmp v0.7.0 github.com/joho/godotenv v1.5.1 diff --git a/go.sum b/go.sum index 001a2222..cada253d 100644 --- a/go.sum +++ b/go.sum @@ -123,11 +123,10 @@ github.com/go-playground/validator/v10 v10.29.0 h1:lQlF5VNJWNlRbRZNeOIkWElR+1LL/ github.com/go-playground/validator/v10 v10.29.0/go.mod h1:D6QxqeMlgIPuT02L66f2ccrZ7AGgHkzKmmTMZhk/Kc4= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/go-vela/sdk-go v0.27.2-0.20251230154052-e1dae5f07d93/go.mod h1:hTkiVuPlurq/r/bAO0e74oDl0YeBfeHGbRI9H9MZVFA= -github.com/go-vela/sdk-go v0.27.2-0.20260203190443-c7ae4e25a718 h1:sFPUqpHVbBC9tviYakmhEtpSgUvbyd2xrAotme84/eE= -github.com/go-vela/sdk-go v0.27.2-0.20260203190443-c7ae4e25a718/go.mod h1:OXizg0doBUj9R4kKMCIx3KNoNxI8hyByjKTlLK86QRI= -github.com/go-vela/server v0.27.3-0.20260203190253-90deb1b0fdd7 h1:iXEnIUxBYWxnKqQfDtJJh3vHYod7KYzKVXDGsP1o37M= -github.com/go-vela/server v0.27.3-0.20260203190253-90deb1b0fdd7/go.mod h1:L83ces3y+5yNaPT5v35zPAzN5le8OROEYoTjldwgT88= +github.com/go-vela/sdk-go v0.27.2-0.20260206210222-679cb21b266d h1:ATViSirEDK74r0qYQkK2CAgwVapJ1lTgP2n703TxerE= +github.com/go-vela/sdk-go v0.27.2-0.20260206210222-679cb21b266d/go.mod h1:0G6Mk4W/TYcIwk/TvSebPtQ65uadS2p/Pa5y0XuHPcw= +github.com/go-vela/server v0.27.3-0.20260206204023-9a8f2f0e09c3 h1:1TNhEFH/XAv1yD3c2QecxYx4BckFw6aaXGYDTWr85Zo= +github.com/go-vela/server v0.27.3-0.20260206204023-9a8f2f0e09c3/go.mod h1:L83ces3y+5yNaPT5v35zPAzN5le8OROEYoTjldwgT88= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.0 h1:EmkZ9RIsX+Uq4DYFowegAuJo8+xdX3T/2dwNPXbxEYE= From 7e69206803269ee64f6439bb603be4f67dbd5f94 Mon Sep 17 00:00:00 2001 From: TimHuynh <163044036+Z003J76_tgt@users.noreply.github.com> Date: Wed, 11 Feb 2026 11:49:08 -0600 Subject: [PATCH 41/63] fix linters --- cmd/vela-worker/exec.go | 6 ++++-- cmd/vela-worker/operate.go | 2 +- executor/linux/outputs.go | 2 ++ runtime/kubernetes/container.go | 4 ++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index e9a04781..d8084a61 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -27,8 +27,10 @@ import ( // //nolint:gocyclo,funlen // ignore cyclomatic complexity and function length func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error { - var err error - var _executor executor.Engine + var ( + err error + _executor executor.Engine + ) // setup the version v := version.New() diff --git a/cmd/vela-worker/operate.go b/cmd/vela-worker/operate.go index 7b8eee48..588c34b1 100644 --- a/cmd/vela-worker/operate.go +++ b/cmd/vela-worker/operate.go @@ -18,7 +18,7 @@ import ( // subprocesses for the operator to poll the // queue and execute Vela pipelines. // -//nolint:funlen,gocyclo // refactor candidate +//nolint:funlen // refactor candidate func (w *Worker) operate(ctx context.Context) error { var err error // create the errgroup for managing operator subprocesses diff --git a/executor/linux/outputs.go b/executor/linux/outputs.go index 0c12c6df..cfce8291 100644 --- a/executor/linux/outputs.go +++ b/executor/linux/outputs.go @@ -239,6 +239,7 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, file if err != nil { return fmt.Errorf("unable to create sts storage client %w with response code %d", err, res.StatusCode) } + if stsStorageClient == nil { return fmt.Errorf("sts storage client is nil") } @@ -277,5 +278,6 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, file return fmt.Errorf("unable to upload object %s: %w", fileName, err) } } + return nil } diff --git a/runtime/kubernetes/container.go b/runtime/kubernetes/container.go index 65bae84e..a2904d44 100644 --- a/runtime/kubernetes/container.go +++ b/runtime/kubernetes/container.go @@ -79,7 +79,7 @@ func (c *client) PollOutputsContainer(_ context.Context, ctn *pipeline.Container // PollFileNames grabs test results and attachments from provided path within a container. // This is a no-op for kubernetes. Pod environments cannot be dynamic. -func (c *client) PollFileNames(_ context.Context, ctn *pipeline.Container, paths []string) ([]string, error) { +func (c *client) PollFileNames(_ context.Context, ctn *pipeline.Container, _ []string) ([]string, error) { c.Logger.Tracef("no-op: gathering test results and attachments from container %s", ctn.ID) return nil, nil @@ -87,7 +87,7 @@ func (c *client) PollFileNames(_ context.Context, ctn *pipeline.Container, paths // PollFileContent captures the content and size of a file from the pipeline container. // This is a no-op for kubernetes. Pod environments cannot be dynamic. -func (c *client) PollFileContent(_ context.Context, ctn *pipeline.Container, path string) (io.Reader, int64, error) { +func (c *client) PollFileContent(_ context.Context, ctn *pipeline.Container, _ string) (io.Reader, int64, error) { c.Logger.Tracef("no-op: gathering test results and attachments from container %s", ctn.ID) return nil, 0, nil From 51b9ff1747528154ec09f04367e7144c6a5730e7 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Wed, 11 Feb 2026 15:19:23 -0600 Subject: [PATCH 42/63] enhance(outputs): use docker client CopyFromContainer for outputs --- docker-compose.yml | 2 +- mock/docker/container.go | 40 ++++++++++++++++++++++- runtime/docker/container.go | 56 +++++++++++++++----------------- runtime/docker/container_test.go | 28 +++++++++++++--- runtime/docker/docker.go | 6 +++- 5 files changed, 95 insertions(+), 37 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0c081b6c..1436bc03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,7 +51,7 @@ services: # https://go-vela.github.io/docs/administration/server/ server: container_name: server - image: target/vela-server:latest + image: server:local networks: - vela environment: diff --git a/mock/docker/container.go b/mock/docker/container.go index 1b1d8e3c..706173d9 100644 --- a/mock/docker/container.go +++ b/mock/docker/container.go @@ -3,6 +3,7 @@ package docker import ( + "archive/tar" "bytes" "context" "encoding/json" @@ -519,7 +520,44 @@ func (c *ContainerService) ContainerStatsOneShot(_ context.Context, _ string) (c // a mocked call to copy content from a Docker container. // // https://pkg.go.dev/github.com/docker/docker/client#Client.CopyFromContainer -func (c *ContainerService) CopyFromContainer(_ context.Context, _ string, _ string) (io.ReadCloser, container.PathStat, error) { +func (c *ContainerService) CopyFromContainer(_ context.Context, ctnID string, path string) (io.ReadCloser, container.PathStat, error) { + if path == "not-found" { + return nil, container.PathStat{}, errdefs.NotFound(fmt.Errorf("error: No such file or directory: %s", path)) + } + + if ctnID == "outputs" { + // create a tar archive in memory with the specified path and content + var buf bytes.Buffer + + tw := tar.NewWriter(&buf) + + content := []byte("key=value") + + hdr := &tar.Header{ + Name: path, + Mode: 0600, + Size: int64(len(content)), + } + + if err := tw.WriteHeader(hdr); err != nil { + return nil, container.PathStat{}, err + } + + if _, err := tw.Write(content); err != nil { + return nil, container.PathStat{}, err + } + + if err := tw.Close(); err != nil { + return nil, container.PathStat{}, err + } + + return io.NopCloser(&buf), container.PathStat{ + Name: path, + Size: int64(len(content)), + Mode: 0600, + }, nil + } + return nil, container.PathStat{}, nil } diff --git a/runtime/docker/container.go b/runtime/docker/container.go index 3f0650d3..cab22928 100644 --- a/runtime/docker/container.go +++ b/runtime/docker/container.go @@ -3,8 +3,10 @@ package docker import ( + "archive/tar" "bytes" "context" + "errors" "fmt" "io" "strings" @@ -294,52 +296,46 @@ func (c *client) WaitContainer(ctx context.Context, ctn *pipeline.Container) err return nil } -// PollOutputsContainer captures the `cat` response for a given path in the Docker volume. +// PollOutputsContainer copies the contents from a given path in the Docker volume. func (c *client) PollOutputsContainer(ctx context.Context, ctn *pipeline.Container, path string) ([]byte, error) { - if len(ctn.Image) == 0 { + if len(ctn.Image) == 0 || len(path) == 0 { return nil, nil } - execConfig := dockerContainerTypes.ExecOptions{ - Tty: true, - Cmd: []string{"sh", "-c", fmt.Sprintf("cat %s", path)}, - AttachStderr: true, - AttachStdout: true, - } - - responseExec, err := c.Docker.ContainerExecCreate(ctx, ctn.ID, execConfig) + // copy file from outputs container + reader, _, err := c.Docker.CopyFromContainer(ctx, ctn.ID, path) if err != nil { - return nil, err - } + // early non-error exit if not found + if errdefs.IsNotFound(err) { + return nil, nil + } - hijackedResponse, err := c.Docker.ContainerExecAttach(ctx, responseExec.ID, dockerContainerTypes.ExecAttachOptions{}) - if err != nil { return nil, err } - defer func() { - if hijackedResponse.Conn != nil { - hijackedResponse.Close() - } - }() + defer reader.Close() - outputStdout := new(bytes.Buffer) - outputStderr := new(bytes.Buffer) + // docker returns tar archive for file + tr := tar.NewReader(reader) - if hijackedResponse.Reader != nil { - _, err := stdcopy.StdCopy(outputStdout, outputStderr, hijackedResponse.Reader) - if err != nil { - c.Logger.Errorf("unable to copy logs for container: %v", err) - } + header, err := tr.Next() + if err != nil { + return nil, err } - if outputStderr.Len() > 0 { - return nil, fmt.Errorf("error: %s", outputStderr.String()) + // check if the file size exceeds the maximum allowed size + if header.Size > MaxOutputsSize { + return nil, fmt.Errorf("outputs file size %d exceeds maximum allowed size of %d", header.Size, MaxOutputsSize) } - data := outputStdout.Bytes() + content := new(bytes.Buffer) + + _, err = io.CopyN(content, tr, header.Size) + if err != nil && !errors.Is(err, io.EOF) { + return nil, err + } - return data, nil + return content.Bytes(), nil } // ctnConfig is a helper function to diff --git a/runtime/docker/container_test.go b/runtime/docker/container_test.go index bd840c0e..d4c70669 100644 --- a/runtime/docker/container_test.go +++ b/runtime/docker/container_test.go @@ -409,11 +409,27 @@ func TestDocker_PollOutputsContainer(t *testing.T) { name string failure bool container *pipeline.Container + path string + wantBytes []byte }{ { - name: "outputs container", - failure: false, - container: _container, + name: "outputs container", + failure: false, + container: &pipeline.Container{ + ID: "outputs", + Image: "alpine:latest", + }, + path: "/vela/outputs/.env", + wantBytes: []byte("key=value"), + }, + { + name: "path not found", + failure: false, + container: &pipeline.Container{ + ID: "outputs", + Image: "alpine:latest", + }, + path: "not-found", }, { name: "no provided outputs container", @@ -425,7 +441,7 @@ func TestDocker_PollOutputsContainer(t *testing.T) { // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err = _engine.PollOutputsContainer(context.Background(), test.container, "") + got, err := _engine.PollOutputsContainer(context.Background(), test.container, test.path) if test.failure { if err == nil { @@ -438,6 +454,10 @@ func TestDocker_PollOutputsContainer(t *testing.T) { if err != nil { t.Errorf("PollOutputs returned err: %v", err) } + + if string(got) != string(test.wantBytes) { + t.Errorf("PollOutputsContainer is %s, want %s", string(got), string(test.wantBytes)) + } }) } } diff --git a/runtime/docker/docker.go b/runtime/docker/docker.go index c2dc7537..2bc9e1fa 100644 --- a/runtime/docker/docker.go +++ b/runtime/docker/docker.go @@ -23,7 +23,11 @@ import ( // * the Docker version of v20.10 has a maximum API version of v1.41 // * to maintain n-1, the API version is pinned to v1.40 // . -const Version = "v1.40" +const ( + Version = "v1.40" + + MaxOutputsSize = 10 * 1024 * 1024 // 10MB +) type config struct { // specifies a list of privileged images to use for the Docker client From 98620e1c5fba76352409e9c5071a2e16a88b8510 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Wed, 11 Feb 2026 15:21:49 -0600 Subject: [PATCH 43/63] revert docker compose --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1436bc03..0c081b6c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,7 +51,7 @@ services: # https://go-vela.github.io/docs/administration/server/ server: container_name: server - image: server:local + image: target/vela-server:latest networks: - vela environment: From 740d0e8b3069d328ada7a582b1bf17c4d01d9d3a Mon Sep 17 00:00:00 2001 From: TimHuynh <163044036+Z003J76_tgt@users.noreply.github.com> Date: Wed, 11 Feb 2026 16:28:32 -0600 Subject: [PATCH 44/63] use docker copyFromContainer --- runtime/docker/artifact.go | 85 ++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 45 deletions(-) diff --git a/runtime/docker/artifact.go b/runtime/docker/artifact.go index 424b3725..8069f222 100644 --- a/runtime/docker/artifact.go +++ b/runtime/docker/artifact.go @@ -3,14 +3,16 @@ package docker import ( + "archive/tar" "bytes" "context" - "encoding/base64" + "errors" "fmt" "io" "path/filepath" "strings" + "github.com/containerd/errdefs" dockerContainerTypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/pkg/stdcopy" @@ -120,66 +122,59 @@ func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, pat func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, path string) (io.Reader, int64, error) { c.Logger.Tracef("gathering test results and attachments from container %s", ctn.ID) - if len(ctn.Image) == 0 { - // return an empty reader instead of nil - return bytes.NewReader(nil), 0, fmt.Errorf("empty container image") + if len(ctn.Image) == 0 || len(path) == 0 { + return nil, 0, nil } - cmd := []string{"sh", "-c", fmt.Sprintf("base64 %s", path)} - execConfig := dockerContainerTypes.ExecOptions{ - Cmd: cmd, - AttachStdout: true, - AttachStderr: false, - Tty: false, + // copy file from outputs container + reader, _, err := c.Docker.CopyFromContainer(ctx, ctn.ID, path) + if err != nil { + c.Logger.Debugf("PollFileContent CopyFromContainer failed for %q: %v", path, err) + // early non-error exit if not found + if errdefs.IsNotFound(err) { + return nil, 0, nil + } + return nil, 0, err } - c.Logger.Infof("executing command for content: %v", execConfig.Cmd) + defer reader.Close() - execID, err := c.Docker.ContainerExecCreate(ctx, ctn.ID, execConfig) - if err != nil { - c.Logger.Debugf("PollFileContent exec-create failed for %q: %v", path, err) - return nil, 0, fmt.Errorf("failed to create exec instance: %w", err) - } + // docker returns a tar archive for the path + tr := tar.NewReader(reader) - resp, err := c.Docker.ContainerExecAttach(ctx, execID.ID, dockerContainerTypes.ExecAttachOptions{}) + header, err := tr.Next() if err != nil { - c.Logger.Debugf("PollFileContent exec-attach failed for %q: %v", path, err) - return nil, 0, fmt.Errorf("failed to attach to exec instance: %w", err) - } - - defer func() { - if resp.Conn != nil { - resp.Close() + // if the tar has no entries or is finished unexpectedly + if errors.Is(err, io.EOF) { + c.Logger.Debugf("PollFileContent: no tar entries for %q", path) + return nil, 0, nil } - }() - - outputStdout := new(bytes.Buffer) - outputStderr := new(bytes.Buffer) + c.Logger.Debugf("PollFileContent tr.Next failed for %q: %v", path, err) - if resp.Reader != nil { - _, err := stdcopy.StdCopy(outputStdout, outputStderr, resp.Reader) - if err != nil { - c.Logger.Errorf("unable to copy logs for container: %v", err) - } + return nil, 0, err } - if outputStderr.Len() > 0 { - return nil, 0, fmt.Errorf("error: %s", outputStderr.String()) + // Ensure the tar entry is a regular file (not dir, symlink, etc.) + if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeRegA { + c.Logger.Debugf("PollFileContent unexpected tar entry type %v for %q", header.Typeflag, path) + + return nil, 0, fmt.Errorf("unexpected tar entry type %v for %q", header.Typeflag, path) } - data := outputStdout.Bytes() + // Read file contents. Use io.ReadAll to avoid dealing with CopyN EOF nuances. + fileBytes, err := io.ReadAll(tr) + if err != nil { + c.Logger.Debugf("PollFileContent ReadAll failed for %q: %v", path, err) + return nil, 0, err + } - // Add logging for empty data in PollFileContent - if len(data) == 0 { + if len(fileBytes) == 0 { c.Logger.Errorf("PollFileContent returned no data for path: %s", path) - return nil, 0, fmt.Errorf("no data returned from base64 command") - } - decoded, err := base64.StdEncoding.DecodeString(string(data)) - if err != nil { - c.Logger.Errorf("unable to decode base64 data: %v", err) - return nil, 0, fmt.Errorf("failed to decode base64 data: %w", err) + return nil, 0, fmt.Errorf("no data returned from container for %q", path) } - return bytes.NewReader(decoded), int64(len(decoded)), nil + // Return a reader and length (use int64 for size) + return bytes.NewReader(fileBytes), int64(len(fileBytes)), nil + } From 750a7aaf0ad98728488534d254f46d35985cce83 Mon Sep 17 00:00:00 2001 From: TimHuynh <163044036+Z003J76_tgt@users.noreply.github.com> Date: Thu, 12 Feb 2026 08:44:01 -0600 Subject: [PATCH 45/63] fix lints --- runtime/docker/artifact.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/runtime/docker/artifact.go b/runtime/docker/artifact.go index 8069f222..1e883da5 100644 --- a/runtime/docker/artifact.go +++ b/runtime/docker/artifact.go @@ -134,6 +134,7 @@ func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, p if errdefs.IsNotFound(err) { return nil, 0, nil } + return nil, 0, err } @@ -147,15 +148,17 @@ func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, p // if the tar has no entries or is finished unexpectedly if errors.Is(err, io.EOF) { c.Logger.Debugf("PollFileContent: no tar entries for %q", path) + return nil, 0, nil } + c.Logger.Debugf("PollFileContent tr.Next failed for %q: %v", path, err) return nil, 0, err } // Ensure the tar entry is a regular file (not dir, symlink, etc.) - if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeRegA { + if header.Typeflag != tar.TypeReg { c.Logger.Debugf("PollFileContent unexpected tar entry type %v for %q", header.Typeflag, path) return nil, 0, fmt.Errorf("unexpected tar entry type %v for %q", header.Typeflag, path) @@ -165,6 +168,7 @@ func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, p fileBytes, err := io.ReadAll(tr) if err != nil { c.Logger.Debugf("PollFileContent ReadAll failed for %q: %v", path, err) + return nil, 0, err } @@ -176,5 +180,4 @@ func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, p // Return a reader and length (use int64 for size) return bytes.NewReader(fileBytes), int64(len(fileBytes)), nil - } From f9af839b7972e9a466c1f18c55c899ae02b8f1aa Mon Sep 17 00:00:00 2001 From: TimHuynh <163044036+Z003J76_tgt@users.noreply.github.com> Date: Mon, 16 Feb 2026 15:10:34 -0600 Subject: [PATCH 46/63] use vela_workspace for finding files --- executor/linux/build.go | 4 +++- executor/linux/outputs.go | 6 +++--- runtime/docker/artifact.go | 7 +++---- runtime/engine.go | 2 +- runtime/kubernetes/container.go | 2 +- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/executor/linux/build.go b/executor/linux/build.go index 4e547ed2..6e5a5655 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/bytedance/gopkg/util/logger" "golang.org/x/sync/errgroup" "github.com/go-vela/sdk-go/vela" @@ -558,7 +559,8 @@ func (c *client) ExecBuild(ctx context.Context) error { } if len(_step.Artifacts.Paths) != 0 { - err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.Artifacts.Paths, c.build) + logger.Debugf("pipeline workspace env: %v", _step.Environment["VELA_WORKSPACE"]) + err := c.outputs.pollFiles(ctx, c.OutputCtn, _step, c.build) if err != nil { c.Logger.Errorf("unable to poll files for artifacts: %v", err) } diff --git a/executor/linux/outputs.go b/executor/linux/outputs.go index cfce8291..d04d960a 100644 --- a/executor/linux/outputs.go +++ b/executor/linux/outputs.go @@ -211,7 +211,7 @@ func (o *outputSvc) poll(ctx context.Context, ctn *pipeline.Container) (map[stri } // pollFiles polls the output for files from the sidecar container. -func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, fileList []string, b *api.Build) error { +func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, _step *pipeline.Container, b *api.Build) error { // exit if outputs container has not been configured if len(ctn.Image) == 0 { return fmt.Errorf("no outputs container configured") @@ -245,13 +245,13 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, file } // grab file paths from the container - filesPath, err := o.client.Runtime.PollFileNames(ctx, ctn, fileList) + filesPath, err := o.client.Runtime.PollFileNames(ctx, ctn, _step) if err != nil { return fmt.Errorf("unable to poll file names: %w", err) } if len(filesPath) == 0 { - return fmt.Errorf("no files found for file list: %v", fileList) + return fmt.Errorf("no files found for file list: %v", _step.Artifacts.Paths) } // process each file found diff --git a/runtime/docker/artifact.go b/runtime/docker/artifact.go index 1e883da5..be887bee 100644 --- a/runtime/docker/artifact.go +++ b/runtime/docker/artifact.go @@ -69,7 +69,7 @@ func (c *client) execContainerLines(ctx context.Context, containerID, cmd string } // PollFileNames searches for files matching the provided patterns within a container. -func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, paths []string) ([]string, error) { +func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, _step *pipeline.Container) ([]string, error) { c.Logger.Tracef("gathering files from container %s", ctn.ID) if ctn.Image == "" { @@ -77,10 +77,11 @@ func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, pat } var results []string + paths := _step.Artifacts.Paths for _, pattern := range paths { // use find command to locate files matching the pattern - cmd := fmt.Sprintf("find / -type f -path '*%s' -print", pattern) + cmd := fmt.Sprintf("find / -type f -path '%s/*%s' -print", _step.Environment["VELA_WORKSPACE"], pattern) c.Logger.Debugf("searching for files with pattern: %s", pattern) lines, err := c.execContainerLines(ctx, ctn.ID, cmd) @@ -113,8 +114,6 @@ func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, pat return results, fmt.Errorf("no matching files found for patterns: %v", paths) } - c.Logger.Infof("found %d files matching patterns", len(results)) - return results, nil } diff --git a/runtime/engine.go b/runtime/engine.go index 1247d45f..31322c71 100644 --- a/runtime/engine.go +++ b/runtime/engine.go @@ -98,7 +98,7 @@ type Engine interface { // PollFileNames defines a function that // captures the artifacts from the pipeline container. - PollFileNames(ctx context.Context, ctn *pipeline.Container, paths []string) ([]string, error) + PollFileNames(ctx context.Context, ctn *pipeline.Container, _step *pipeline.Container) ([]string, error) // PollFileContent defines a function that // captures the content and size of a file from the pipeline container. PollFileContent(ctx context.Context, ctn *pipeline.Container, path string) (io.Reader, int64, error) diff --git a/runtime/kubernetes/container.go b/runtime/kubernetes/container.go index a2904d44..cbe9d7ff 100644 --- a/runtime/kubernetes/container.go +++ b/runtime/kubernetes/container.go @@ -79,7 +79,7 @@ func (c *client) PollOutputsContainer(_ context.Context, ctn *pipeline.Container // PollFileNames grabs test results and attachments from provided path within a container. // This is a no-op for kubernetes. Pod environments cannot be dynamic. -func (c *client) PollFileNames(_ context.Context, ctn *pipeline.Container, _ []string) ([]string, error) { +func (c *client) PollFileNames(_ context.Context, ctn *pipeline.Container, _ *pipeline.Container) ([]string, error) { c.Logger.Tracef("no-op: gathering test results and attachments from container %s", ctn.ID) return nil, nil From 2c7a65193f3daf5ccec97320e08f77564ced0b48 Mon Sep 17 00:00:00 2001 From: TimHuynh <163044036+Z003J76_tgt@users.noreply.github.com> Date: Mon, 16 Feb 2026 15:20:41 -0600 Subject: [PATCH 47/63] use vela_workspace for finding files --- runtime/docker/artifact.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/docker/artifact.go b/runtime/docker/artifact.go index be887bee..26eab7f5 100644 --- a/runtime/docker/artifact.go +++ b/runtime/docker/artifact.go @@ -81,7 +81,7 @@ func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, _st for _, pattern := range paths { // use find command to locate files matching the pattern - cmd := fmt.Sprintf("find / -type f -path '%s/*%s' -print", _step.Environment["VELA_WORKSPACE"], pattern) + cmd := fmt.Sprintf("find %s -type f -path '*%s' -print", _step.Environment["VELA_WORKSPACE"], pattern) c.Logger.Debugf("searching for files with pattern: %s", pattern) lines, err := c.execContainerLines(ctx, ctn.ID, cmd) From 99c9e0314c05ca3ede2b9c2c00232972dbfaa9e1 Mon Sep 17 00:00:00 2001 From: TimHuynh <163044036+Z003J76_tgt@users.noreply.github.com> Date: Mon, 16 Feb 2026 16:27:25 -0600 Subject: [PATCH 48/63] fix lints --- executor/linux/build.go | 2 -- runtime/docker/artifact.go | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/executor/linux/build.go b/executor/linux/build.go index 6e5a5655..9893df5b 100644 --- a/executor/linux/build.go +++ b/executor/linux/build.go @@ -10,7 +10,6 @@ import ( "sync" "time" - "github.com/bytedance/gopkg/util/logger" "golang.org/x/sync/errgroup" "github.com/go-vela/sdk-go/vela" @@ -559,7 +558,6 @@ func (c *client) ExecBuild(ctx context.Context) error { } if len(_step.Artifacts.Paths) != 0 { - logger.Debugf("pipeline workspace env: %v", _step.Environment["VELA_WORKSPACE"]) err := c.outputs.pollFiles(ctx, c.OutputCtn, _step, c.build) if err != nil { c.Logger.Errorf("unable to poll files for artifacts: %v", err) diff --git a/runtime/docker/artifact.go b/runtime/docker/artifact.go index 26eab7f5..e9303306 100644 --- a/runtime/docker/artifact.go +++ b/runtime/docker/artifact.go @@ -77,6 +77,7 @@ func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, _st } var results []string + paths := _step.Artifacts.Paths for _, pattern := range paths { From 528ade2d29f896b30ed23ae71db7cc68a2919223 Mon Sep 17 00:00:00 2001 From: TimHuynh <163044036+Z003J76_tgt@users.noreply.github.com> Date: Thu, 19 Feb 2026 10:50:00 -0600 Subject: [PATCH 49/63] remove filetypes --- go.mod | 4 ++-- go.sum | 8 ++++---- runtime/docker/artifact.go | 22 +--------------------- 3 files changed, 7 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index a1c95a39..c8af179b 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,8 @@ require ( github.com/docker/docker v28.5.1+incompatible github.com/docker/go-units v0.5.0 github.com/gin-gonic/gin v1.11.0 - github.com/go-vela/sdk-go v0.27.2-0.20260206210222-679cb21b266d - github.com/go-vela/server v0.27.3-0.20260206204023-9a8f2f0e09c3 + github.com/go-vela/sdk-go v0.27.2-0.20260211182039-bc08b39ec6b8 + github.com/go-vela/server v0.27.3-0.20260219164020-7897fa1bf40f github.com/golang-jwt/jwt/v5 v5.3.0 github.com/google/go-cmp v0.7.0 github.com/joho/godotenv v1.5.1 diff --git a/go.sum b/go.sum index cada253d..643286b4 100644 --- a/go.sum +++ b/go.sum @@ -123,10 +123,10 @@ github.com/go-playground/validator/v10 v10.29.0 h1:lQlF5VNJWNlRbRZNeOIkWElR+1LL/ github.com/go-playground/validator/v10 v10.29.0/go.mod h1:D6QxqeMlgIPuT02L66f2ccrZ7AGgHkzKmmTMZhk/Kc4= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/go-vela/sdk-go v0.27.2-0.20260206210222-679cb21b266d h1:ATViSirEDK74r0qYQkK2CAgwVapJ1lTgP2n703TxerE= -github.com/go-vela/sdk-go v0.27.2-0.20260206210222-679cb21b266d/go.mod h1:0G6Mk4W/TYcIwk/TvSebPtQ65uadS2p/Pa5y0XuHPcw= -github.com/go-vela/server v0.27.3-0.20260206204023-9a8f2f0e09c3 h1:1TNhEFH/XAv1yD3c2QecxYx4BckFw6aaXGYDTWr85Zo= -github.com/go-vela/server v0.27.3-0.20260206204023-9a8f2f0e09c3/go.mod h1:L83ces3y+5yNaPT5v35zPAzN5le8OROEYoTjldwgT88= +github.com/go-vela/sdk-go v0.27.2-0.20260211182039-bc08b39ec6b8 h1:hC6onCgXujPeZ0B0huaxNnYf2go0TAZag7fSi40ic48= +github.com/go-vela/sdk-go v0.27.2-0.20260211182039-bc08b39ec6b8/go.mod h1:n11qaLdTi57g1y/i5PonAuh+L9RKHGzGUg/4C7PJF9M= +github.com/go-vela/server v0.27.3-0.20260219164020-7897fa1bf40f h1:cViDebS0LyhmEUZ50bJXEXzJNjpsWYHIPUOIosZP7To= +github.com/go-vela/server v0.27.3-0.20260219164020-7897fa1bf40f/go.mod h1:o/Kfk0wIrJNVmLVmUyg+coejEDqhfF5F2hWVexi5duA= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.0 h1:EmkZ9RIsX+Uq4DYFowegAuJo8+xdX3T/2dwNPXbxEYE= diff --git a/runtime/docker/artifact.go b/runtime/docker/artifact.go index e9303306..cccbead4 100644 --- a/runtime/docker/artifact.go +++ b/runtime/docker/artifact.go @@ -17,21 +17,8 @@ import ( "github.com/docker/docker/pkg/stdcopy" "github.com/go-vela/server/compiler/types/pipeline" - "github.com/go-vela/server/constants" ) -// isAllowedExt returns true if ext (".xml", ".png", etc.) is in your allow-list. -func isAllowedExt(ext string) bool { - ext = strings.ToLower(ext) - for _, a := range constants.AllAllowedExtensions { - if ext == a { - return true - } - } - - return false -} - // execContainerLines runs `sh -c cmd` in the named container and // returns its stdout split by newline (error if anything on stderr). func (c *client) execContainerLines(ctx context.Context, containerID, cmd string) ([]string, error) { @@ -77,7 +64,7 @@ func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, _st } var results []string - + paths := _step.Artifacts.Paths for _, pattern := range paths { @@ -99,13 +86,6 @@ func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, _st continue } - // check if file extension is allowed - ext := strings.ToLower(filepath.Ext(filePath)) - if !isAllowedExt(ext) { - c.Logger.Debugf("skipping file %s (extension %s not allowed)", filePath, ext) - continue - } - c.Logger.Debugf("accepted file: %s", filePath) results = append(results, filePath) } From c8dc836ae0d02af2b0418b1367283d3f088fadd9 Mon Sep 17 00:00:00 2001 From: TimHuynh <163044036+Z003J76_tgt@users.noreply.github.com> Date: Mon, 23 Feb 2026 09:22:34 -0600 Subject: [PATCH 50/63] fix gosec lint --- .golangci.yml | 7 ++++++- executor/linux/linux.go | 2 +- executor/local/local.go | 2 +- mock/docker/docker.go | 2 +- runtime/docker/docker.go | 4 ++-- runtime/kubernetes/kubernetes.go | 2 +- runtime/kubernetes/mock.go | 2 +- 7 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index a3976d8a..7aede90e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -54,7 +54,12 @@ linters: wsl_v5: allow-first-in-block: true allow-whole-block: false - branch-max-lines: 2 + branch-max-lines: 2 + gosec: + excludes: + - G101 + - G117 + - G705 exclusions: generated: lax presets: diff --git a/executor/linux/linux.go b/executor/linux/linux.go index 70502d85..83f54dcf 100644 --- a/executor/linux/linux.go +++ b/executor/linux/linux.go @@ -88,7 +88,7 @@ func Equal(a, b *client) bool { // New returns an Executor implementation that integrates with a Linux instance. // -//nolint:revive // ignore unexported type as it is intentional + func New(opts ...Opt) (*client, error) { // create new Linux client c := new(client) diff --git a/executor/local/local.go b/executor/local/local.go index 33f88a55..337fe24d 100644 --- a/executor/local/local.go +++ b/executor/local/local.go @@ -77,7 +77,7 @@ func Equal(a, b *client) bool { // New returns an Executor implementation that integrates with the local system. // -//nolint:revive // ignore unexported type as it is intentional + func New(opts ...Opt) (*client, error) { // create new local client c := new(client) diff --git a/mock/docker/docker.go b/mock/docker/docker.go index 4f7923c0..0007b504 100644 --- a/mock/docker/docker.go +++ b/mock/docker/docker.go @@ -21,7 +21,7 @@ const Version = "v1.44" // New returns a client that is capable of handling // Docker client calls and returning stub responses. // -//nolint:revive // ignore unexported type as it is intentional + func New() (*mock, error) { return &mock{ ConfigService: ConfigService{}, diff --git a/runtime/docker/docker.go b/runtime/docker/docker.go index 2bc9e1fa..58e9619c 100644 --- a/runtime/docker/docker.go +++ b/runtime/docker/docker.go @@ -49,7 +49,7 @@ type client struct { // New returns an Engine implementation that // integrates with a Docker runtime. // -//nolint:revive // ignore returning unexported client + func New(opts ...ClientOpt) (*client, error) { // create new Docker client c := new(client) @@ -102,7 +102,7 @@ func New(opts ...ClientOpt) (*client, error) { // // This function is intended for running tests only. // -//nolint:revive // ignore returning unexported client + func NewMock(opts ...ClientOpt) (*client, error) { // create new Docker runtime client c, err := New(opts...) diff --git a/runtime/kubernetes/kubernetes.go b/runtime/kubernetes/kubernetes.go index 400c31b5..d1bba371 100644 --- a/runtime/kubernetes/kubernetes.go +++ b/runtime/kubernetes/kubernetes.go @@ -51,7 +51,7 @@ type client struct { // New returns an Engine implementation that // integrates with a Kubernetes runtime. // -//nolint:revive // ignore returning unexported client + func New(opts ...ClientOpt) (*client, error) { // create new Kubernetes client c := new(client) diff --git a/runtime/kubernetes/mock.go b/runtime/kubernetes/mock.go index 61361d89..0841a970 100644 --- a/runtime/kubernetes/mock.go +++ b/runtime/kubernetes/mock.go @@ -23,7 +23,7 @@ import ( // // This function is intended for running tests only. // -//nolint:revive // ignore returning unexported client + func NewMock(_pod *v1.Pod, opts ...ClientOpt) (*client, error) { // create new Kubernetes client c := new(client) From bf83ce316e38d837fccb5e04ea2d407b3c9d3a14 Mon Sep 17 00:00:00 2001 From: TimHuynh <163044036+Z003J76_tgt@users.noreply.github.com> Date: Tue, 24 Feb 2026 08:12:47 -0600 Subject: [PATCH 51/63] fix linters --- .golangci.yml | 4 ++++ executor/context.go | 2 +- executor/context_test.go | 6 +++--- runtime/context.go | 2 +- runtime/context_test.go | 6 +++--- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 7aede90e..f8fc27fc 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -60,6 +60,10 @@ linters: - G101 - G117 - G705 + revive: + rules: + - name: var-naming + disabled: true exclusions: generated: lax presets: diff --git a/executor/context.go b/executor/context.go index 1439a7bf..5f90580a 100644 --- a/executor/context.go +++ b/executor/context.go @@ -52,7 +52,7 @@ func FromGinContext(c *gin.Context) Engine { func WithContext(c context.Context, e Engine) context.Context { // set the executor Engine in the context.Context // - //nolint:revive,staticcheck // ignore using string with context value + //nolint:staticcheck // ignore using string with context value return context.WithValue(c, key, e) } diff --git a/executor/context_test.go b/executor/context_test.go index a2aef53f..f25d6646 100644 --- a/executor/context_test.go +++ b/executor/context_test.go @@ -50,7 +50,7 @@ func TestExecutor_FromContext(t *testing.T) { }{ { name: "valid executor in context", - //nolint:staticcheck,revive // ignore using string with context value + //nolint:staticcheck // ignore using string with context value context: context.WithValue(context.Background(), key, _engine), want: _engine, }, @@ -61,7 +61,7 @@ func TestExecutor_FromContext(t *testing.T) { }, { name: "invalid executor in context", - //nolint:staticcheck,revive // ignore using string with context value + //nolint:staticcheck // ignore using string with context value context: context.WithValue(context.Background(), key, "foo"), want: nil, }, @@ -174,7 +174,7 @@ func TestExecutor_WithContext(t *testing.T) { t.Errorf("unable to create linux engine: %v", err) } - //nolint:staticcheck,revive // ignore using string with context value + //nolint:staticcheck // ignore using string with context value want := context.WithValue(context.Background(), key, _engine) // run test diff --git a/runtime/context.go b/runtime/context.go index 72ce1d95..9dcb14ac 100644 --- a/runtime/context.go +++ b/runtime/context.go @@ -52,7 +52,7 @@ func FromGinContext(c *gin.Context) Engine { func WithContext(c context.Context, e Engine) context.Context { // set the runtime Engine in the context.Context // - //nolint:revive,staticcheck // ignore using string with context value + //nolint:staticcheck // ignore using string with context value return context.WithValue(c, key, e) } diff --git a/runtime/context_test.go b/runtime/context_test.go index 8f1c7fe3..4aecc130 100644 --- a/runtime/context_test.go +++ b/runtime/context_test.go @@ -29,7 +29,7 @@ func TestRuntime_FromContext(t *testing.T) { }{ { name: "valid runtime in context", - //nolint:staticcheck,revive // ignore using string with context value + //nolint:staticcheck // ignore using string with context value context: context.WithValue(context.Background(), key, _engine), want: _engine, }, @@ -40,7 +40,7 @@ func TestRuntime_FromContext(t *testing.T) { }, { name: "invalid runtime in context", - //nolint:staticcheck,revive // ignore using string with context value + //nolint:staticcheck // ignore using string with context value context: context.WithValue(context.Background(), key, "foo"), want: nil, }, @@ -119,7 +119,7 @@ func TestRuntime_WithContext(t *testing.T) { t.Errorf("unable to create runtime engine: %v", err) } - //nolint:staticcheck,revive // ignore using string with context value + //nolint:staticcheck // ignore using string with context value want := context.WithValue(context.Background(), key, _engine) // run test From 86a6a7a8b902efcbd19780180b80e96235706bd2 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Tue, 24 Feb 2026 10:53:53 -0600 Subject: [PATCH 52/63] use CopyFromContainer for finding matching files, update docker compose, add tests, support stages --- cmd/vela-worker/exec.go | 2 - docker-compose.yml | 50 ++++++++- executor/linux/stage.go | 7 ++ mock/docker/container.go | 127 +++++++++++++++++---- runtime/docker/artifact.go | 100 ++++++++--------- runtime/docker/artifact_test.go | 184 ++++++++++++++++++++++++++++++- runtime/docker/container_test.go | 2 +- 7 files changed, 397 insertions(+), 75 deletions(-) diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index d8084a61..2c5c8c68 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -147,8 +147,6 @@ func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error // prepare pipeline by hydrating container ID values based on build information p.Prepare(item.Build.GetRepo().GetOrg(), item.Build.GetRepo().GetName(), item.Build.GetNumber(), false) - logrus.Debugf("setting up exec client with scm token %s with expiration %d", p.Token, p.TokenExp) - // setup exec client with scm token and build token execBuildClient, err = setupExecClient(w.Config.Server, bt.GetToken(), p.Token, p.TokenExp, item.Build) if err != nil { diff --git a/docker-compose.yml b/docker-compose.yml index 0c081b6c..9eedaf88 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: VELA_SERVER_SECRET: 'zB7mrKDTZqNeNTD8z47yG4DHywspAh' WORKER_ADDR: 'http://worker:8080' WORKER_CHECK_IN: 2m - VELA_EXECUTOR_OUTPUTS_IMAGE: 'alpine:latest' + VELA_EXECUTOR_OUTPUTS_IMAGE: 'busybox:stable' restart: always ports: - "8081:8080" @@ -83,6 +83,17 @@ services: VELA_DISABLE_WEBHOOK_VALIDATION: 'true' VELA_ENABLE_SECURE_COOKIE: 'false' VELA_REPO_ALLOWLIST: '*' + VELA_SCHEDULE_ALLOWLIST: '*' + VELA_OTEL_TRACING_ENABLE: true + VELA_OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 + VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 100 + VELA_STORAGE_ENABLE: true + VELA_STORAGE_DRIVER: minio + VELA_STORAGE_ADDRESS: 'http://minio:9001' # Address of the MinIO server + VELA_STORAGE_ACCESS_KEY: minioadmin + VELA_STORAGE_SECRET_KEY: minioadmin + VELA_STORAGE_USE_SSL: 'false' + VELA_STORAGE_BUCKET: vela env_file: - .env restart: always @@ -161,5 +172,42 @@ services: cap_add: - IPC_LOCK + # The `minio` compose service hosts the MinIO server instance. + # + # This component is used for storing build artifacts. + # + # https://min.io/ + minio: + container_name: minio + image: minio/minio + restart: always + ports: + - '9001:9001' + - '9002:9002' + networks: + - vela + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin + command: minio server --address ":9001" --console-address ":9002" /data + + # The `minio-setup` compose service is used for setting up the MinIO server instance. + # + # This component is used for creating the bucket and setting permissions. + minio-setup: + image: minio/mc + container_name: minio-setup + depends_on: + - minio + networks: + - vela + entrypoint: > + /bin/sh -c ' + until mc alias set local http://minio:9001 minioadmin minioadmin; do sleep 1; done; + mc mb --ignore-existing local/vela; + mc anonymous set none local/vela; + ' + restart: "no" + networks: vela: \ No newline at end of file diff --git a/executor/linux/stage.go b/executor/linux/stage.go index 42c7bf06..4581cfba 100644 --- a/executor/linux/stage.go +++ b/executor/linux/stage.go @@ -198,6 +198,13 @@ func (c *client) ExecStage(ctx context.Context, s *pipeline.Stage, m *sync.Map) if _step.ExitCode != 0 && !_step.Ruleset.Continue { stageStatus = constants.StatusFailure } + + if len(_step.Artifacts.Paths) != 0 { + err := c.outputs.pollFiles(ctx, c.OutputCtn, _step, c.build) + if err != nil { + c.Logger.Errorf("unable to poll files for artifacts: %v", err) + } + } } return nil diff --git a/mock/docker/container.go b/mock/docker/container.go index c67fc917..c9c72598 100644 --- a/mock/docker/container.go +++ b/mock/docker/container.go @@ -521,7 +521,7 @@ func (c *ContainerService) ContainerStatsOneShot(_ context.Context, _ string) (c // // https://pkg.go.dev/github.com/docker/docker/client#Client.CopyFromContainer func (c *ContainerService) CopyFromContainer(_ context.Context, _ string, path string) (io.ReadCloser, container.PathStat, error) { - if path == "not-found" { + if strings.Contains(path, "not-found") { return nil, container.PathStat{}, errdefs.NotFound(fmt.Errorf("error: No such file or directory: %s", path)) } // create a tar archive in memory with the specified path and content @@ -529,31 +529,118 @@ func (c *ContainerService) CopyFromContainer(_ context.Context, _ string, path s tw := tar.NewWriter(&buf) - content := []byte("key=value") + // Handle workspace paths for artifact searches + if strings.Contains(path, "workspace") { + // Create nested directory structure with files + entries := []struct { + name string + isDir bool + content []byte + }{ + {"artifacts", true, nil}, + {"artifacts/test_results", true, nil}, + {"artifacts/test_results/alpha.txt", false, []byte("test alpha content")}, + {"artifacts/test_results/beta.txt", false, []byte("test beta content")}, + {"artifacts/build_results", true, nil}, + {"artifacts/build_results/alpha.txt", false, []byte("build alpha content")}, + {"artifacts/build_results/beta.txt", false, []byte("build beta content")}, + } - hdr := &tar.Header{ - Name: path, - Mode: 0600, - Size: int64(len(content)), - } + for _, entry := range entries { + var hdr *tar.Header + if entry.isDir { + hdr = &tar.Header{ + Name: entry.name + "/", + Mode: 0755, + Typeflag: tar.TypeDir, + } + } else { + hdr = &tar.Header{ + Name: entry.name, + Mode: 0644, + Size: int64(len(entry.content)), + Typeflag: tar.TypeReg, + } + } + + if err := tw.WriteHeader(hdr); err != nil { + return nil, container.PathStat{}, err + } + + if !entry.isDir { + if _, err := tw.Write(entry.content); err != nil { + return nil, container.PathStat{}, err + } + } + } - if err := tw.WriteHeader(hdr); err != nil { - return nil, container.PathStat{}, err - } + if err := tw.Close(); err != nil { + return nil, container.PathStat{}, err + } - if _, err := tw.Write(content); err != nil { - return nil, container.PathStat{}, err + return io.NopCloser(&buf), container.PathStat{ + Name: "artifacts", + Mode: 0755, + }, nil } - if err := tw.Close(); err != nil { - return nil, container.PathStat{}, err - } + switch path { + case "outputs": + content := []byte("key=value") + + hdr := &tar.Header{ + Name: path, + Mode: 0600, + Size: int64(len(content)), + } + + if err := tw.WriteHeader(hdr); err != nil { + return nil, container.PathStat{}, err + } + + if _, err := tw.Write(content); err != nil { + return nil, container.PathStat{}, err + } - return io.NopCloser(&buf), container.PathStat{ - Name: path, - Size: int64(len(content)), - Mode: 0600, - }, nil + if err := tw.Close(); err != nil { + return nil, container.PathStat{}, err + } + + return io.NopCloser(&buf), container.PathStat{ + Name: path, + Size: int64(len(content)), + Mode: 0600, + }, nil + + default: + // Default case: return simple file content + content := []byte("results") + + hdr := &tar.Header{ + Name: path, + Mode: 0644, + Size: int64(len(content)), + Typeflag: tar.TypeReg, + } + + if err := tw.WriteHeader(hdr); err != nil { + return nil, container.PathStat{}, err + } + + if _, err := tw.Write(content); err != nil { + return nil, container.PathStat{}, err + } + + if err := tw.Close(); err != nil { + return nil, container.PathStat{}, err + } + + return io.NopCloser(&buf), container.PathStat{ + Name: path, + Size: int64(len(content)), + Mode: 0644, + }, nil + } } // CopyToContainer is a helper function to simulate diff --git a/runtime/docker/artifact.go b/runtime/docker/artifact.go index cccbead4..dac99a85 100644 --- a/runtime/docker/artifact.go +++ b/runtime/docker/artifact.go @@ -13,48 +13,10 @@ import ( "strings" "github.com/containerd/errdefs" - dockerContainerTypes "github.com/docker/docker/api/types/container" - "github.com/docker/docker/pkg/stdcopy" "github.com/go-vela/server/compiler/types/pipeline" ) -// execContainerLines runs `sh -c cmd` in the named container and -// returns its stdout split by newline (error if anything on stderr). -func (c *client) execContainerLines(ctx context.Context, containerID, cmd string) ([]string, error) { - execConfig := dockerContainerTypes.ExecOptions{ - Tty: true, - Cmd: []string{"sh", "-c", cmd}, - AttachStdout: true, - AttachStderr: true, - } - - resp, err := c.Docker.ContainerExecCreate(ctx, containerID, execConfig) - if err != nil { - return nil, fmt.Errorf("create exec: %w", err) - } - - attach, err := c.Docker.ContainerExecAttach(ctx, resp.ID, dockerContainerTypes.ExecAttachOptions{}) - if err != nil { - return nil, fmt.Errorf("attach exec: %w", err) - } - - defer attach.Close() - - var outBuf, errBuf bytes.Buffer - if _, err := stdcopy.StdCopy(&outBuf, &errBuf, attach.Reader); err != nil { - return nil, fmt.Errorf("copy exec output: %w", err) - } - - if errBuf.Len() > 0 { - return nil, fmt.Errorf("exec error: %s", errBuf.String()) - } - - lines := strings.Split(strings.TrimSpace(outBuf.String()), "\n") - - return lines, nil -} - // PollFileNames searches for files matching the provided patterns within a container. func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, _step *pipeline.Container) ([]string, error) { c.Logger.Tracef("gathering files from container %s", ctn.ID) @@ -65,30 +27,51 @@ func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, _st var results []string + seen := make(map[string]bool) paths := _step.Artifacts.Paths + workspacePrefix := _step.Environment["VELA_WORKSPACE"] + "/" for _, pattern := range paths { - // use find command to locate files matching the pattern - cmd := fmt.Sprintf("find %s -type f -path '*%s' -print", _step.Environment["VELA_WORKSPACE"], pattern) - c.Logger.Debugf("searching for files with pattern: %s", pattern) + searchDir := extractSearchDir(workspacePrefix + pattern) - lines, err := c.execContainerLines(ctx, ctn.ID, cmd) + reader, _, err := c.Docker.CopyFromContainer(ctx, ctn.ID, searchDir) if err != nil { - return nil, fmt.Errorf("failed to search for pattern %q: %w", pattern, err) + if errdefs.IsNotFound(err) { + c.Logger.Debugf("search directory %q not found in container: %v", searchDir, err) + continue + } + + return nil, fmt.Errorf("copy from container for search dir %q: %w", searchDir, err) } - c.Logger.Tracef("found %d candidates for pattern %s", len(lines), pattern) + tr := tar.NewReader(reader) + + for { + header, err := tr.Next() + if err != nil { + if errors.Is(err, io.EOF) { + break + } - // process each found file - for _, line := range lines { - filePath := filepath.Clean(strings.TrimSpace(line)) - if filePath == "" { + reader.Close() + + return nil, fmt.Errorf("read tar entry for search dir %q: %w", searchDir, err) + } + + if header.Typeflag != tar.TypeReg { continue } - c.Logger.Debugf("accepted file: %s", filePath) - results = append(results, filePath) + filePath := filepath.Clean(header.Name) + + matched, err := filepath.Match(pattern, filePath) + if err == nil && matched && !seen[filePath] { + results = append(results, workspacePrefix+filePath) + seen[filePath] = true + } } + + reader.Close() } if len(results) == 0 { @@ -98,6 +81,23 @@ func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, _st return results, nil } +// extractSearchDir extracts a search directory from a glob pattern. +func extractSearchDir(pattern string) string { + // if no wildcard, determine directory + idx := strings.IndexAny(pattern, "*?[") + if idx == -1 { + return filepath.Dir(pattern) + } + + // determine directory before wildcard + dir := filepath.Dir(pattern[:idx]) + if dir == "" || dir == "." { + return "/" + } + + return dir +} + // PollFileContent retrieves the content and size of a file inside a container. func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, path string) (io.Reader, int64, error) { c.Logger.Tracef("gathering test results and attachments from container %s", ctn.ID) diff --git a/runtime/docker/artifact_test.go b/runtime/docker/artifact_test.go index 19116929..f15642bb 100644 --- a/runtime/docker/artifact_test.go +++ b/runtime/docker/artifact_test.go @@ -2,4 +2,186 @@ package docker -// TODO +import ( + "context" + "io" + "testing" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestDocker_PollFileNames(t *testing.T) { + // setup Docker + _engine, err := NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + name string + failure bool + container *pipeline.Container + step *pipeline.Container + wantFiles int + }{ + { + name: "artifacts directory search", + failure: false, + container: &pipeline.Container{ + ID: "artifacts-container", + Image: "alpine:latest", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{"artifacts/*/*.txt"}, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/vela/workspace", + }, + }, + wantFiles: 4, // alpha.txt and beta.txt in test_results and build_results + }, + { + name: "directory not found", + failure: true, + container: &pipeline.Container{ + ID: "artifacts-container", + Image: "alpine:latest", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{"artifacts/*.txt"}, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/not-found", + }, + }, + }, + { + name: "empty container image", + failure: false, + container: &pipeline.Container{ + ID: "no-image", + Image: "", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{"*.txt"}, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/vela/workspace", + }, + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := _engine.PollFileNames(context.Background(), test.container, test.step) + + if test.failure { + if err == nil { + t.Errorf("PollFileNames should have returned err") + } + + return // continue to next test + } + + if err != nil { + t.Errorf("PollFileNames returned err: %v", err) + } + + if test.wantFiles > 0 && len(got) != test.wantFiles { + t.Errorf("PollFileNames returned %d files, want %d", len(got), test.wantFiles) + } + }) + } +} + +func TestDocker_PollFileContent(t *testing.T) { + // setup Docker + _engine, err := NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + name string + failure bool + container *pipeline.Container + path string + wantBytes []byte + }{ + { + name: "file content from default path", + failure: false, + container: &pipeline.Container{ + ID: "content-container", + Image: "alpine:latest", + }, + path: "/vela/artifacts/test_results/alpha.txt", + wantBytes: []byte("results"), + }, + { + name: "path not found", + failure: false, + container: &pipeline.Container{ + ID: "content-container", + Image: "alpine:latest", + }, + path: "not-found", + }, + { + name: "empty container image", + failure: false, + container: new(pipeline.Container), + path: "/some/path", + }, + { + name: "empty path", + failure: false, + container: &pipeline.Container{ + ID: "content-container", + Image: "alpine:latest", + }, + path: "", + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + reader, size, err := _engine.PollFileContent(context.Background(), test.container, test.path) + + if test.failure { + if err == nil { + t.Errorf("PollFileContent should have returned err") + } + + return // continue to next test + } + + if err != nil { + t.Errorf("PollFileContent returned err: %v", err) + } + + if test.wantBytes != nil { + got, err := io.ReadAll(reader) + if err != nil { + t.Errorf("failed to read content: %v", err) + } + + if string(got) != string(test.wantBytes) { + t.Errorf("PollFileContent is %s, want %s", string(got), string(test.wantBytes)) + } + + if size != int64(len(test.wantBytes)) { + t.Errorf("PollFileContent size is %d, want %d", size, int64(len(test.wantBytes))) + } + } + }) + } +} diff --git a/runtime/docker/container_test.go b/runtime/docker/container_test.go index d4c70669..dea2aaae 100644 --- a/runtime/docker/container_test.go +++ b/runtime/docker/container_test.go @@ -419,7 +419,7 @@ func TestDocker_PollOutputsContainer(t *testing.T) { ID: "outputs", Image: "alpine:latest", }, - path: "/vela/outputs/.env", + path: "outputs", wantBytes: []byte("key=value"), }, { From ef707cfc5aa899ef4fb7fc0cde24e9bf1d37305e Mon Sep 17 00:00:00 2001 From: ecrupper Date: Tue, 24 Feb 2026 13:05:49 -0600 Subject: [PATCH 53/63] Revert "use CopyFromContainer for finding matching files, update docker compose, add tests, support stages" This reverts commit 86a6a7a8b902efcbd19780180b80e96235706bd2. --- cmd/vela-worker/exec.go | 2 + docker-compose.yml | 50 +-------- executor/linux/stage.go | 7 -- mock/docker/container.go | 127 ++++----------------- runtime/docker/artifact.go | 100 ++++++++--------- runtime/docker/artifact_test.go | 184 +------------------------------ runtime/docker/container_test.go | 2 +- 7 files changed, 75 insertions(+), 397 deletions(-) diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index 2c5c8c68..d8084a61 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -147,6 +147,8 @@ func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error // prepare pipeline by hydrating container ID values based on build information p.Prepare(item.Build.GetRepo().GetOrg(), item.Build.GetRepo().GetName(), item.Build.GetNumber(), false) + logrus.Debugf("setting up exec client with scm token %s with expiration %d", p.Token, p.TokenExp) + // setup exec client with scm token and build token execBuildClient, err = setupExecClient(w.Config.Server, bt.GetToken(), p.Token, p.TokenExp, item.Build) if err != nil { diff --git a/docker-compose.yml b/docker-compose.yml index 9eedaf88..0c081b6c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: VELA_SERVER_SECRET: 'zB7mrKDTZqNeNTD8z47yG4DHywspAh' WORKER_ADDR: 'http://worker:8080' WORKER_CHECK_IN: 2m - VELA_EXECUTOR_OUTPUTS_IMAGE: 'busybox:stable' + VELA_EXECUTOR_OUTPUTS_IMAGE: 'alpine:latest' restart: always ports: - "8081:8080" @@ -83,17 +83,6 @@ services: VELA_DISABLE_WEBHOOK_VALIDATION: 'true' VELA_ENABLE_SECURE_COOKIE: 'false' VELA_REPO_ALLOWLIST: '*' - VELA_SCHEDULE_ALLOWLIST: '*' - VELA_OTEL_TRACING_ENABLE: true - VELA_OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 - VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 100 - VELA_STORAGE_ENABLE: true - VELA_STORAGE_DRIVER: minio - VELA_STORAGE_ADDRESS: 'http://minio:9001' # Address of the MinIO server - VELA_STORAGE_ACCESS_KEY: minioadmin - VELA_STORAGE_SECRET_KEY: minioadmin - VELA_STORAGE_USE_SSL: 'false' - VELA_STORAGE_BUCKET: vela env_file: - .env restart: always @@ -172,42 +161,5 @@ services: cap_add: - IPC_LOCK - # The `minio` compose service hosts the MinIO server instance. - # - # This component is used for storing build artifacts. - # - # https://min.io/ - minio: - container_name: minio - image: minio/minio - restart: always - ports: - - '9001:9001' - - '9002:9002' - networks: - - vela - environment: - - MINIO_ROOT_USER=minioadmin - - MINIO_ROOT_PASSWORD=minioadmin - command: minio server --address ":9001" --console-address ":9002" /data - - # The `minio-setup` compose service is used for setting up the MinIO server instance. - # - # This component is used for creating the bucket and setting permissions. - minio-setup: - image: minio/mc - container_name: minio-setup - depends_on: - - minio - networks: - - vela - entrypoint: > - /bin/sh -c ' - until mc alias set local http://minio:9001 minioadmin minioadmin; do sleep 1; done; - mc mb --ignore-existing local/vela; - mc anonymous set none local/vela; - ' - restart: "no" - networks: vela: \ No newline at end of file diff --git a/executor/linux/stage.go b/executor/linux/stage.go index 4581cfba..42c7bf06 100644 --- a/executor/linux/stage.go +++ b/executor/linux/stage.go @@ -198,13 +198,6 @@ func (c *client) ExecStage(ctx context.Context, s *pipeline.Stage, m *sync.Map) if _step.ExitCode != 0 && !_step.Ruleset.Continue { stageStatus = constants.StatusFailure } - - if len(_step.Artifacts.Paths) != 0 { - err := c.outputs.pollFiles(ctx, c.OutputCtn, _step, c.build) - if err != nil { - c.Logger.Errorf("unable to poll files for artifacts: %v", err) - } - } } return nil diff --git a/mock/docker/container.go b/mock/docker/container.go index c9c72598..c67fc917 100644 --- a/mock/docker/container.go +++ b/mock/docker/container.go @@ -521,7 +521,7 @@ func (c *ContainerService) ContainerStatsOneShot(_ context.Context, _ string) (c // // https://pkg.go.dev/github.com/docker/docker/client#Client.CopyFromContainer func (c *ContainerService) CopyFromContainer(_ context.Context, _ string, path string) (io.ReadCloser, container.PathStat, error) { - if strings.Contains(path, "not-found") { + if path == "not-found" { return nil, container.PathStat{}, errdefs.NotFound(fmt.Errorf("error: No such file or directory: %s", path)) } // create a tar archive in memory with the specified path and content @@ -529,118 +529,31 @@ func (c *ContainerService) CopyFromContainer(_ context.Context, _ string, path s tw := tar.NewWriter(&buf) - // Handle workspace paths for artifact searches - if strings.Contains(path, "workspace") { - // Create nested directory structure with files - entries := []struct { - name string - isDir bool - content []byte - }{ - {"artifacts", true, nil}, - {"artifacts/test_results", true, nil}, - {"artifacts/test_results/alpha.txt", false, []byte("test alpha content")}, - {"artifacts/test_results/beta.txt", false, []byte("test beta content")}, - {"artifacts/build_results", true, nil}, - {"artifacts/build_results/alpha.txt", false, []byte("build alpha content")}, - {"artifacts/build_results/beta.txt", false, []byte("build beta content")}, - } - - for _, entry := range entries { - var hdr *tar.Header - if entry.isDir { - hdr = &tar.Header{ - Name: entry.name + "/", - Mode: 0755, - Typeflag: tar.TypeDir, - } - } else { - hdr = &tar.Header{ - Name: entry.name, - Mode: 0644, - Size: int64(len(entry.content)), - Typeflag: tar.TypeReg, - } - } - - if err := tw.WriteHeader(hdr); err != nil { - return nil, container.PathStat{}, err - } - - if !entry.isDir { - if _, err := tw.Write(entry.content); err != nil { - return nil, container.PathStat{}, err - } - } - } + content := []byte("key=value") - if err := tw.Close(); err != nil { - return nil, container.PathStat{}, err - } - - return io.NopCloser(&buf), container.PathStat{ - Name: "artifacts", - Mode: 0755, - }, nil + hdr := &tar.Header{ + Name: path, + Mode: 0600, + Size: int64(len(content)), } - switch path { - case "outputs": - content := []byte("key=value") - - hdr := &tar.Header{ - Name: path, - Mode: 0600, - Size: int64(len(content)), - } - - if err := tw.WriteHeader(hdr); err != nil { - return nil, container.PathStat{}, err - } - - if _, err := tw.Write(content); err != nil { - return nil, container.PathStat{}, err - } - - if err := tw.Close(); err != nil { - return nil, container.PathStat{}, err - } - - return io.NopCloser(&buf), container.PathStat{ - Name: path, - Size: int64(len(content)), - Mode: 0600, - }, nil - - default: - // Default case: return simple file content - content := []byte("results") - - hdr := &tar.Header{ - Name: path, - Mode: 0644, - Size: int64(len(content)), - Typeflag: tar.TypeReg, - } - - if err := tw.WriteHeader(hdr); err != nil { - return nil, container.PathStat{}, err - } - - if _, err := tw.Write(content); err != nil { - return nil, container.PathStat{}, err - } + if err := tw.WriteHeader(hdr); err != nil { + return nil, container.PathStat{}, err + } - if err := tw.Close(); err != nil { - return nil, container.PathStat{}, err - } + if _, err := tw.Write(content); err != nil { + return nil, container.PathStat{}, err + } - return io.NopCloser(&buf), container.PathStat{ - Name: path, - Size: int64(len(content)), - Mode: 0644, - }, nil + if err := tw.Close(); err != nil { + return nil, container.PathStat{}, err } + + return io.NopCloser(&buf), container.PathStat{ + Name: path, + Size: int64(len(content)), + Mode: 0600, + }, nil } // CopyToContainer is a helper function to simulate diff --git a/runtime/docker/artifact.go b/runtime/docker/artifact.go index dac99a85..cccbead4 100644 --- a/runtime/docker/artifact.go +++ b/runtime/docker/artifact.go @@ -13,10 +13,48 @@ import ( "strings" "github.com/containerd/errdefs" + dockerContainerTypes "github.com/docker/docker/api/types/container" + "github.com/docker/docker/pkg/stdcopy" "github.com/go-vela/server/compiler/types/pipeline" ) +// execContainerLines runs `sh -c cmd` in the named container and +// returns its stdout split by newline (error if anything on stderr). +func (c *client) execContainerLines(ctx context.Context, containerID, cmd string) ([]string, error) { + execConfig := dockerContainerTypes.ExecOptions{ + Tty: true, + Cmd: []string{"sh", "-c", cmd}, + AttachStdout: true, + AttachStderr: true, + } + + resp, err := c.Docker.ContainerExecCreate(ctx, containerID, execConfig) + if err != nil { + return nil, fmt.Errorf("create exec: %w", err) + } + + attach, err := c.Docker.ContainerExecAttach(ctx, resp.ID, dockerContainerTypes.ExecAttachOptions{}) + if err != nil { + return nil, fmt.Errorf("attach exec: %w", err) + } + + defer attach.Close() + + var outBuf, errBuf bytes.Buffer + if _, err := stdcopy.StdCopy(&outBuf, &errBuf, attach.Reader); err != nil { + return nil, fmt.Errorf("copy exec output: %w", err) + } + + if errBuf.Len() > 0 { + return nil, fmt.Errorf("exec error: %s", errBuf.String()) + } + + lines := strings.Split(strings.TrimSpace(outBuf.String()), "\n") + + return lines, nil +} + // PollFileNames searches for files matching the provided patterns within a container. func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, _step *pipeline.Container) ([]string, error) { c.Logger.Tracef("gathering files from container %s", ctn.ID) @@ -27,51 +65,30 @@ func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, _st var results []string - seen := make(map[string]bool) paths := _step.Artifacts.Paths - workspacePrefix := _step.Environment["VELA_WORKSPACE"] + "/" for _, pattern := range paths { - searchDir := extractSearchDir(workspacePrefix + pattern) + // use find command to locate files matching the pattern + cmd := fmt.Sprintf("find %s -type f -path '*%s' -print", _step.Environment["VELA_WORKSPACE"], pattern) + c.Logger.Debugf("searching for files with pattern: %s", pattern) - reader, _, err := c.Docker.CopyFromContainer(ctx, ctn.ID, searchDir) + lines, err := c.execContainerLines(ctx, ctn.ID, cmd) if err != nil { - if errdefs.IsNotFound(err) { - c.Logger.Debugf("search directory %q not found in container: %v", searchDir, err) - continue - } - - return nil, fmt.Errorf("copy from container for search dir %q: %w", searchDir, err) + return nil, fmt.Errorf("failed to search for pattern %q: %w", pattern, err) } - tr := tar.NewReader(reader) - - for { - header, err := tr.Next() - if err != nil { - if errors.Is(err, io.EOF) { - break - } + c.Logger.Tracef("found %d candidates for pattern %s", len(lines), pattern) - reader.Close() - - return nil, fmt.Errorf("read tar entry for search dir %q: %w", searchDir, err) - } - - if header.Typeflag != tar.TypeReg { + // process each found file + for _, line := range lines { + filePath := filepath.Clean(strings.TrimSpace(line)) + if filePath == "" { continue } - filePath := filepath.Clean(header.Name) - - matched, err := filepath.Match(pattern, filePath) - if err == nil && matched && !seen[filePath] { - results = append(results, workspacePrefix+filePath) - seen[filePath] = true - } + c.Logger.Debugf("accepted file: %s", filePath) + results = append(results, filePath) } - - reader.Close() } if len(results) == 0 { @@ -81,23 +98,6 @@ func (c *client) PollFileNames(ctx context.Context, ctn *pipeline.Container, _st return results, nil } -// extractSearchDir extracts a search directory from a glob pattern. -func extractSearchDir(pattern string) string { - // if no wildcard, determine directory - idx := strings.IndexAny(pattern, "*?[") - if idx == -1 { - return filepath.Dir(pattern) - } - - // determine directory before wildcard - dir := filepath.Dir(pattern[:idx]) - if dir == "" || dir == "." { - return "/" - } - - return dir -} - // PollFileContent retrieves the content and size of a file inside a container. func (c *client) PollFileContent(ctx context.Context, ctn *pipeline.Container, path string) (io.Reader, int64, error) { c.Logger.Tracef("gathering test results and attachments from container %s", ctn.ID) diff --git a/runtime/docker/artifact_test.go b/runtime/docker/artifact_test.go index f15642bb..19116929 100644 --- a/runtime/docker/artifact_test.go +++ b/runtime/docker/artifact_test.go @@ -2,186 +2,4 @@ package docker -import ( - "context" - "io" - "testing" - - "github.com/go-vela/server/compiler/types/pipeline" -) - -func TestDocker_PollFileNames(t *testing.T) { - // setup Docker - _engine, err := NewMock() - if err != nil { - t.Errorf("unable to create runtime engine: %v", err) - } - - // setup tests - tests := []struct { - name string - failure bool - container *pipeline.Container - step *pipeline.Container - wantFiles int - }{ - { - name: "artifacts directory search", - failure: false, - container: &pipeline.Container{ - ID: "artifacts-container", - Image: "alpine:latest", - }, - step: &pipeline.Container{ - Artifacts: pipeline.Artifacts{ - Paths: []string{"artifacts/*/*.txt"}, - }, - Environment: map[string]string{ - "VELA_WORKSPACE": "/vela/workspace", - }, - }, - wantFiles: 4, // alpha.txt and beta.txt in test_results and build_results - }, - { - name: "directory not found", - failure: true, - container: &pipeline.Container{ - ID: "artifacts-container", - Image: "alpine:latest", - }, - step: &pipeline.Container{ - Artifacts: pipeline.Artifacts{ - Paths: []string{"artifacts/*.txt"}, - }, - Environment: map[string]string{ - "VELA_WORKSPACE": "/not-found", - }, - }, - }, - { - name: "empty container image", - failure: false, - container: &pipeline.Container{ - ID: "no-image", - Image: "", - }, - step: &pipeline.Container{ - Artifacts: pipeline.Artifacts{ - Paths: []string{"*.txt"}, - }, - Environment: map[string]string{ - "VELA_WORKSPACE": "/vela/workspace", - }, - }, - }, - } - - // run tests - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - got, err := _engine.PollFileNames(context.Background(), test.container, test.step) - - if test.failure { - if err == nil { - t.Errorf("PollFileNames should have returned err") - } - - return // continue to next test - } - - if err != nil { - t.Errorf("PollFileNames returned err: %v", err) - } - - if test.wantFiles > 0 && len(got) != test.wantFiles { - t.Errorf("PollFileNames returned %d files, want %d", len(got), test.wantFiles) - } - }) - } -} - -func TestDocker_PollFileContent(t *testing.T) { - // setup Docker - _engine, err := NewMock() - if err != nil { - t.Errorf("unable to create runtime engine: %v", err) - } - - // setup tests - tests := []struct { - name string - failure bool - container *pipeline.Container - path string - wantBytes []byte - }{ - { - name: "file content from default path", - failure: false, - container: &pipeline.Container{ - ID: "content-container", - Image: "alpine:latest", - }, - path: "/vela/artifacts/test_results/alpha.txt", - wantBytes: []byte("results"), - }, - { - name: "path not found", - failure: false, - container: &pipeline.Container{ - ID: "content-container", - Image: "alpine:latest", - }, - path: "not-found", - }, - { - name: "empty container image", - failure: false, - container: new(pipeline.Container), - path: "/some/path", - }, - { - name: "empty path", - failure: false, - container: &pipeline.Container{ - ID: "content-container", - Image: "alpine:latest", - }, - path: "", - }, - } - - // run tests - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - reader, size, err := _engine.PollFileContent(context.Background(), test.container, test.path) - - if test.failure { - if err == nil { - t.Errorf("PollFileContent should have returned err") - } - - return // continue to next test - } - - if err != nil { - t.Errorf("PollFileContent returned err: %v", err) - } - - if test.wantBytes != nil { - got, err := io.ReadAll(reader) - if err != nil { - t.Errorf("failed to read content: %v", err) - } - - if string(got) != string(test.wantBytes) { - t.Errorf("PollFileContent is %s, want %s", string(got), string(test.wantBytes)) - } - - if size != int64(len(test.wantBytes)) { - t.Errorf("PollFileContent size is %d, want %d", size, int64(len(test.wantBytes))) - } - } - }) - } -} +// TODO diff --git a/runtime/docker/container_test.go b/runtime/docker/container_test.go index dea2aaae..d4c70669 100644 --- a/runtime/docker/container_test.go +++ b/runtime/docker/container_test.go @@ -419,7 +419,7 @@ func TestDocker_PollOutputsContainer(t *testing.T) { ID: "outputs", Image: "alpine:latest", }, - path: "outputs", + path: "/vela/outputs/.env", wantBytes: []byte("key=value"), }, { From 0beebefd2ca565f6948b66b0c88bec512ba6c262 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Tue, 24 Feb 2026 13:08:10 -0600 Subject: [PATCH 54/63] add back goodies from previous commit --- cmd/vela-worker/exec.go | 2 -- docker-compose.yml | 48 +++++++++++++++++++++++++++++++++++++++++ executor/linux/stage.go | 7 ++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index d8084a61..2c5c8c68 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -147,8 +147,6 @@ func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error // prepare pipeline by hydrating container ID values based on build information p.Prepare(item.Build.GetRepo().GetOrg(), item.Build.GetRepo().GetName(), item.Build.GetNumber(), false) - logrus.Debugf("setting up exec client with scm token %s with expiration %d", p.Token, p.TokenExp) - // setup exec client with scm token and build token execBuildClient, err = setupExecClient(w.Config.Server, bt.GetToken(), p.Token, p.TokenExp, item.Build) if err != nil { diff --git a/docker-compose.yml b/docker-compose.yml index 0c081b6c..5b464577 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,6 +83,17 @@ services: VELA_DISABLE_WEBHOOK_VALIDATION: 'true' VELA_ENABLE_SECURE_COOKIE: 'false' VELA_REPO_ALLOWLIST: '*' + VELA_SCHEDULE_ALLOWLIST: '*' + VELA_OTEL_TRACING_ENABLE: true + VELA_OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 + VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 100 + VELA_STORAGE_ENABLE: true + VELA_STORAGE_DRIVER: minio + VELA_STORAGE_ADDRESS: 'http://minio:9001' # Address of the MinIO server + VELA_STORAGE_ACCESS_KEY: minioadmin + VELA_STORAGE_SECRET_KEY: minioadmin + VELA_STORAGE_USE_SSL: 'false' + VELA_STORAGE_BUCKET: vela env_file: - .env restart: always @@ -161,5 +172,42 @@ services: cap_add: - IPC_LOCK + # The `minio` compose service hosts the MinIO server instance. + # + # This component is used for storing build artifacts. + # + # https://min.io/ + minio: + container_name: minio + image: minio/minio + restart: always + ports: + - '9001:9001' + - '9002:9002' + networks: + - vela + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin + command: minio server --address ":9001" --console-address ":9002" /data + + # The `minio-setup` compose service is used for setting up the MinIO server instance. + # + # This component is used for creating the bucket and setting permissions. + minio-setup: + image: minio/mc + container_name: minio-setup + depends_on: + - minio + networks: + - vela + entrypoint: > + /bin/sh -c ' + until mc alias set local http://minio:9001 minioadmin minioadmin; do sleep 1; done; + mc mb --ignore-existing local/vela; + mc anonymous set none local/vela; + ' + restart: "no" + networks: vela: \ No newline at end of file diff --git a/executor/linux/stage.go b/executor/linux/stage.go index 42c7bf06..4581cfba 100644 --- a/executor/linux/stage.go +++ b/executor/linux/stage.go @@ -198,6 +198,13 @@ func (c *client) ExecStage(ctx context.Context, s *pipeline.Stage, m *sync.Map) if _step.ExitCode != 0 && !_step.Ruleset.Continue { stageStatus = constants.StatusFailure } + + if len(_step.Artifacts.Paths) != 0 { + err := c.outputs.pollFiles(ctx, c.OutputCtn, _step, c.build) + if err != nil { + c.Logger.Errorf("unable to poll files for artifacts: %v", err) + } + } } return nil From 4dc187042ca04031451b51c3f4e078699df5b908 Mon Sep 17 00:00:00 2001 From: Tim Huynh Date: Tue, 24 Feb 2026 14:13:24 -0600 Subject: [PATCH 55/63] add tests --- mock/docker/container.go | 125 ++++++++++++- runtime/docker/artifact_test.go | 306 +++++++++++++++++++++++++++++++- 2 files changed, 425 insertions(+), 6 deletions(-) diff --git a/mock/docker/container.go b/mock/docker/container.go index c67fc917..496daad1 100644 --- a/mock/docker/container.go +++ b/mock/docker/container.go @@ -4,12 +4,14 @@ package docker import ( "archive/tar" + "bufio" "bytes" "context" "encoding/json" "errors" "fmt" "io" + "net" "strings" "time" @@ -24,6 +26,19 @@ import ( v1 "github.com/opencontainers/image-spec/specs-go/v1" ) +// mockConn is a mock implementation of net.Conn for testing purposes. +// It provides no-op implementations of all net.Conn methods. +type mockConn struct{} + +func (m *mockConn) Read(b []byte) (n int, err error) { return 0, io.EOF } +func (m *mockConn) Write(b []byte) (n int, err error) { return len(b), nil } +func (m *mockConn) Close() error { return nil } +func (m *mockConn) LocalAddr() net.Addr { return nil } +func (m *mockConn) RemoteAddr() net.Addr { return nil } +func (m *mockConn) SetDeadline(t time.Time) error { return nil } +func (m *mockConn) SetReadDeadline(t time.Time) error { return nil } +func (m *mockConn) SetWriteDeadline(t time.Time) error { return nil } + // ContainerService implements all the container // related functions for the Docker mock. type ContainerService struct{} @@ -106,8 +121,51 @@ func (c *ContainerService) ContainerDiff(_ context.Context, _ string) ([]contain // running inside a Docker container. // // https://pkg.go.dev/github.com/docker/docker/client#Client.ContainerExecAttach -func (c *ContainerService) ContainerExecAttach(_ context.Context, _ string, _ container.ExecAttachOptions) (types.HijackedResponse, error) { - return types.HijackedResponse{}, nil +func (c *ContainerService) ContainerExecAttach(_ context.Context, execID string, _ container.ExecAttachOptions) (types.HijackedResponse, error) { + // create a buffer to hold mock output + var buf bytes.Buffer + + // write mock stdout using stdcopy format + stdoutWriter := stdcopy.NewStdWriter(&buf, stdcopy.Stdout) + stderrWriter := stdcopy.NewStdWriter(&buf, stdcopy.Stderr) + + // check for specific test scenarios based on execID + if strings.Contains(execID, "error") { + // write to stderr to simulate an error + _, _ = stderrWriter.Write([]byte("mock exec error")) + } else if strings.Contains(execID, "multiline") { + // write multiple lines for testing + _, _ = stdoutWriter.Write([]byte("line1\nline2\nline3")) + } else if strings.Contains(execID, "artifacts-find") { + // simulate find command for artifacts - return file paths + _, _ = stdoutWriter.Write([]byte("/vela/artifacts/test_results/alpha.txt\n/vela/artifacts/test_results/beta.txt\n/vela/artifacts/build_results/alpha.txt\n/vela/artifacts/build_results/beta.txt")) + } else if strings.Contains(execID, "test-results-xml") { + // simulate find command for test results XML files + _, _ = stdoutWriter.Write([]byte("/vela/workspace/test-results/junit.xml\n/vela/workspace/test-results/report.xml")) + } else if strings.Contains(execID, "cypress-screenshots") { + // simulate find command for cypress screenshots + _, _ = stdoutWriter.Write([]byte("/vela/workspace/cypress/screenshots/test1/screenshot1.png\n/vela/workspace/cypress/screenshots/test1/screenshot2.png\n/vela/workspace/cypress/screenshots/test2/error.png")) + } else if strings.Contains(execID, "cypress-videos") { + // simulate find command for cypress videos + _, _ = stdoutWriter.Write([]byte("/vela/workspace/cypress/videos/test1.mp4\n/vela/workspace/cypress/videos/test2.mp4")) + } else if strings.Contains(execID, "cypress-all") { + // simulate find command for all cypress artifacts (screenshots + videos) + _, _ = stdoutWriter.Write([]byte("/vela/workspace/cypress/screenshots/test1/screenshot1.png\n/vela/workspace/cypress/screenshots/test2/error.png\n/vela/workspace/cypress/videos/test1.mp4\n/vela/workspace/cypress/videos/test2.mp4")) + } else if strings.Contains(execID, "not-found") { + // simulate path not found + _, _ = stderrWriter.Write([]byte("find: '/not-found': No such file or directory")) + } else { + // default mock output + _, _ = stdoutWriter.Write([]byte("mock exec output")) + } + + // create a HijackedResponse with the mock data + response := types.HijackedResponse{ + Reader: bufio.NewReader(&buf), + Conn: &mockConn{}, // Use mock connection to avoid nil pointer dereference + } + + return response, nil } // ContainerExecCreate is a helper function to simulate @@ -115,8 +173,53 @@ func (c *ContainerService) ContainerExecAttach(_ context.Context, _ string, _ co // Docker container. // // https://pkg.go.dev/github.com/docker/docker/client#Client.ContainerExecCreate -func (c *ContainerService) ContainerExecCreate(_ context.Context, _ string, _ container.ExecOptions) (container.ExecCreateResponse, error) { - return container.ExecCreateResponse{}, nil +func (c *ContainerService) ContainerExecCreate(_ context.Context, ctn string, config container.ExecOptions) (container.ExecCreateResponse, error) { + // verify a container was provided + if len(ctn) == 0 { + return container.ExecCreateResponse{}, errors.New("no container provided") + } + + // check if the container is not found + if strings.Contains(ctn, "notfound") || strings.Contains(ctn, "not-found") { + //nolint:staticcheck // message is capitalized to match Docker messages + return container.ExecCreateResponse{}, errdefs.NotFound(fmt.Errorf("Error: No such container: %s", ctn)) + } + + // create exec ID based on command for testing scenarios + execID := stringid.GenerateRandomID() + + // check command for specific test scenarios + if len(config.Cmd) > 0 { + cmdStr := strings.Join(config.Cmd, " ") + if strings.Contains(cmdStr, "error") { + execID = "exec-error-" + execID + } else if strings.Contains(cmdStr, "multiline") { + execID = "exec-multiline-" + execID + } else if strings.Contains(cmdStr, "find") { + // For artifact file search commands + if strings.Contains(cmdStr, "/not-found") { + execID = "exec-not-found-" + execID + } else if strings.Contains(cmdStr, "artifacts") { + execID = "exec-artifacts-find-" + execID + } else if strings.Contains(cmdStr, "test-results") && strings.Contains(cmdStr, ".xml") { + execID = "exec-test-results-xml-" + execID + } else if strings.Contains(cmdStr, "cypress/screenshots") && strings.Contains(cmdStr, ".png") { + execID = "exec-cypress-screenshots-" + execID + } else if strings.Contains(cmdStr, "cypress/videos") && strings.Contains(cmdStr, ".mp4") { + execID = "exec-cypress-videos-" + execID + } else if strings.Contains(cmdStr, "cypress") { + // Generic cypress pattern for combined searches + execID = "exec-cypress-all-" + execID + } + } + } + + // create response object to return + response := container.ExecCreateResponse{ + ID: execID, + } + + return response, nil } // ContainerExecInspect is a helper function to simulate @@ -529,7 +632,19 @@ func (c *ContainerService) CopyFromContainer(_ context.Context, _ string, path s tw := tar.NewWriter(&buf) - content := []byte("key=value") + // Determine content based on path for test scenarios + var content []byte + if strings.Contains(path, "artifacts") && strings.Contains(path, "alpha.txt") { + content = []byte("results") + } else if strings.Contains(path, "test-results") && strings.Contains(path, ".xml") { + content = []byte("") + } else if strings.Contains(path, "cypress/screenshots") && strings.Contains(path, ".png") { + content = []byte("PNG_BINARY_DATA") + } else if strings.Contains(path, "cypress/videos") && strings.Contains(path, ".mp4") { + content = []byte("MP4_BINARY_DATA") + } else { + content = []byte("key=value") + } hdr := &tar.Header{ Name: path, diff --git a/runtime/docker/artifact_test.go b/runtime/docker/artifact_test.go index 19116929..48887f6a 100644 --- a/runtime/docker/artifact_test.go +++ b/runtime/docker/artifact_test.go @@ -2,4 +2,308 @@ package docker -// TODO +import ( + "context" + "io" + "testing" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestDocker_PollFileNames(t *testing.T) { + // setup Docker + _engine, err := NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + name string + failure bool + container *pipeline.Container + step *pipeline.Container + wantFiles int + }{ + { + name: "artifacts directory search", + failure: false, + container: &pipeline.Container{ + ID: "artifacts-container", + Image: "alpine:latest", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{"artifacts/*/*.txt"}, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/vela/workspace", + }, + }, + wantFiles: 4, // alpha.txt and beta.txt in test_results and build_results + }, + { + name: "test-results XML files", + failure: false, + container: &pipeline.Container{ + ID: "test-results-container", + Image: "alpine:latest", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{"test-results/*.xml"}, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/vela/workspace", + }, + }, + wantFiles: 2, // junit.xml and report.xml + }, + { + name: "cypress screenshots PNG files", + failure: false, + container: &pipeline.Container{ + ID: "cypress-screenshots-container", + Image: "cypress/browsers:latest", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{"cypress/screenshots/**/*.png"}, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/vela/workspace", + }, + }, + wantFiles: 3, // screenshot1.png, screenshot2.png, error.png + }, + { + name: "cypress videos MP4 files", + failure: false, + container: &pipeline.Container{ + ID: "cypress-videos-container", + Image: "cypress/browsers:latest", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{"cypress/videos/**/*.mp4"}, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/vela/workspace", + }, + }, + wantFiles: 2, // test1.mp4, test2.mp4 + }, + { + name: "multiple cypress patterns", + failure: false, + container: &pipeline.Container{ + ID: "cypress-all-container", + Image: "cypress/browsers:latest", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{ + "cypress/screenshots/**/*.png", + "cypress/videos/**/*.mp4", + }, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/vela/workspace", + }, + }, + wantFiles: 5, // 3 PNG screenshots + 2 MP4 videos + }, + { + name: "combined test-results and cypress artifacts", + failure: false, + container: &pipeline.Container{ + ID: "combined-artifacts-container", + Image: "cypress/browsers:latest", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{ + "test-results/*.xml", + "cypress/screenshots/**/*.png", + "cypress/videos/**/*.mp4", + }, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/vela/workspace", + }, + }, + wantFiles: 7, // 2 XML + 3 PNG + 2 MP4 + }, + { + name: "directory not found", + failure: true, + container: &pipeline.Container{ + ID: "artifacts-container", + Image: "alpine:latest", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{"artifacts/*.txt"}, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/not-found", + }, + }, + }, + { + name: "empty container image", + failure: false, + container: &pipeline.Container{ + ID: "no-image", + Image: "", + }, + step: &pipeline.Container{ + Artifacts: pipeline.Artifacts{ + Paths: []string{"*.txt"}, + }, + Environment: map[string]string{ + "VELA_WORKSPACE": "/vela/workspace", + }, + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := _engine.PollFileNames(context.Background(), test.container, test.step) + + if test.failure { + if err == nil { + t.Errorf("PollFileNames should have returned err") + } + + return // continue to next test + } + + if err != nil { + t.Errorf("PollFileNames returned err: %v", err) + } + + if test.wantFiles > 0 && len(got) != test.wantFiles { + t.Errorf("PollFileNames returned %d files, want %d", len(got), test.wantFiles) + } + }) + } +} + +func TestDocker_PollFileContent(t *testing.T) { + // setup Docker + _engine, err := NewMock() + if err != nil { + t.Errorf("unable to create runtime engine: %v", err) + } + + // setup tests + tests := []struct { + name string + failure bool + container *pipeline.Container + path string + wantBytes []byte + }{ + { + name: "file content from default path", + failure: false, + container: &pipeline.Container{ + ID: "content-container", + Image: "alpine:latest", + }, + path: "/vela/artifacts/test_results/alpha.txt", + wantBytes: []byte("results"), + }, + { + name: "test-results XML file content", + failure: false, + container: &pipeline.Container{ + ID: "content-container", + Image: "alpine:latest", + }, + path: "/vela/workspace/test-results/junit.xml", + wantBytes: []byte(""), + }, + { + name: "cypress screenshot PNG file", + failure: false, + container: &pipeline.Container{ + ID: "content-container", + Image: "cypress/browsers:latest", + }, + path: "/vela/workspace/cypress/screenshots/test1/screenshot1.png", + wantBytes: []byte("PNG_BINARY_DATA"), + }, + { + name: "cypress video MP4 file", + failure: false, + container: &pipeline.Container{ + ID: "content-container", + Image: "cypress/browsers:latest", + }, + path: "/vela/workspace/cypress/videos/test1.mp4", + wantBytes: []byte("MP4_BINARY_DATA"), + }, + { + name: "path not found", + failure: false, + container: &pipeline.Container{ + ID: "content-container", + Image: "alpine:latest", + }, + path: "not-found", + }, + { + name: "empty container image", + failure: false, + container: new(pipeline.Container), + path: "/some/path", + }, + { + name: "empty path", + failure: false, + container: &pipeline.Container{ + ID: "content-container", + Image: "alpine:latest", + }, + path: "", + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + reader, size, err := _engine.PollFileContent(context.Background(), test.container, test.path) + + if test.failure { + if err == nil { + t.Errorf("PollFileContent should have returned err") + } + + return // continue to next test + } + + if err != nil { + t.Errorf("PollFileContent returned err: %v", err) + } + + if test.wantBytes != nil { + got, err := io.ReadAll(reader) + if err != nil { + t.Errorf("failed to read content: %v", err) + } + + if string(got) != string(test.wantBytes) { + t.Errorf("PollFileContent is %s, want %s", string(got), string(test.wantBytes)) + } + + if size != int64(len(test.wantBytes)) { + t.Errorf("PollFileContent size is %d, want %d", size, int64(len(test.wantBytes))) + } + } + }) + } +} From c7126fb3a92be49a47dd128795b9855de8bd8a95 Mon Sep 17 00:00:00 2001 From: Tim Huynh Date: Wed, 25 Feb 2026 12:36:21 -0600 Subject: [PATCH 56/63] make clean --- .golangci.yml | 9 +++------ go.mod | 3 +-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 14dd3ea8..94e5a737 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -125,7 +125,8 @@ linters: gosec: excludes: - G101 # hardcoded credentials - - G117 # exported field of "Secret" style + - G117 + - G705 # exported field of "Secret" style # https://github.com/client9/misspell misspell: locale: US @@ -158,11 +159,7 @@ linters: allow-first-in-block: true allow-whole-block: false branch-max-lines: 2 - gosec: - excludes: - - G101 - - G117 - - G705 + exclusions: generated: lax presets: diff --git a/go.mod b/go.mod index 047ef0eb..04435e5a 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/go-cmp v0.7.0 github.com/joho/godotenv v1.5.1 - github.com/minio/minio-go/v7 v7.0.89 + github.com/minio/minio-go/v7 v7.0.98 github.com/opencontainers/image-spec v1.1.1 github.com/prometheus/client_golang v1.23.2 github.com/sirupsen/logrus v1.9.4 @@ -62,7 +62,6 @@ require ( github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.98 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect From 3f5f4de44d6536dde26cbfb6873558a3a89a9143 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Wed, 25 Feb 2026 12:44:11 -0600 Subject: [PATCH 57/63] linter --- executor/local/service.go | 2 -- executor/local/step.go | 2 -- mock/docker/container.go | 1 - 3 files changed, 5 deletions(-) diff --git a/executor/local/service.go b/executor/local/service.go index 34186a03..aded9ace 100644 --- a/executor/local/service.go +++ b/executor/local/service.go @@ -119,8 +119,6 @@ func (c *client) StreamService(ctx context.Context, ctn *pipeline.Container) err // scan entire container output for scanner.Scan() { // ensure we output to stdout - // - //nolint:gosec // false positive fmt.Fprintln(c.stdout, _pattern, scanner.Text()) } diff --git a/executor/local/step.go b/executor/local/step.go index dc70b054..ac12a79d 100644 --- a/executor/local/step.go +++ b/executor/local/step.go @@ -195,8 +195,6 @@ func (c *client) StreamStep(ctx context.Context, ctn *pipeline.Container) error // scan entire container output for scanner.Scan() { // ensure we output to stdout - // - //nolint:gosec // false positive fmt.Fprintln(c.stdout, _pattern, scanner.Text()) } diff --git a/mock/docker/container.go b/mock/docker/container.go index 07b298a5..2f2ac60a 100644 --- a/mock/docker/container.go +++ b/mock/docker/container.go @@ -181,7 +181,6 @@ func (c *ContainerService) ContainerExecCreate(_ context.Context, ctn string, co // check if the container is not found if strings.Contains(ctn, "notfound") || strings.Contains(ctn, "not-found") { - //nolint:staticcheck // message is capitalized to match Docker messages return container.ExecCreateResponse{}, errdefs.NotFound(fmt.Errorf("Error: No such container: %s", ctn)) } From dc5601da053c21b1f2ad7d48d57725a9591d1cc5 Mon Sep 17 00:00:00 2001 From: Tim Huynh Date: Wed, 25 Feb 2026 12:44:59 -0600 Subject: [PATCH 58/63] fix linters --- executor/local/service.go | 2 +- executor/local/step.go | 2 +- mock/docker/container.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/executor/local/service.go b/executor/local/service.go index 34186a03..6d9c51b9 100644 --- a/executor/local/service.go +++ b/executor/local/service.go @@ -120,7 +120,7 @@ func (c *client) StreamService(ctx context.Context, ctn *pipeline.Container) err for scanner.Scan() { // ensure we output to stdout // - //nolint:gosec // false positive + fmt.Fprintln(c.stdout, _pattern, scanner.Text()) } diff --git a/executor/local/step.go b/executor/local/step.go index dc70b054..0e2bf56f 100644 --- a/executor/local/step.go +++ b/executor/local/step.go @@ -196,7 +196,7 @@ func (c *client) StreamStep(ctx context.Context, ctn *pipeline.Container) error for scanner.Scan() { // ensure we output to stdout // - //nolint:gosec // false positive + fmt.Fprintln(c.stdout, _pattern, scanner.Text()) } diff --git a/mock/docker/container.go b/mock/docker/container.go index 07b298a5..10e5d8fb 100644 --- a/mock/docker/container.go +++ b/mock/docker/container.go @@ -181,7 +181,7 @@ func (c *ContainerService) ContainerExecCreate(_ context.Context, ctn string, co // check if the container is not found if strings.Contains(ctn, "notfound") || strings.Contains(ctn, "not-found") { - //nolint:staticcheck // message is capitalized to match Docker messages + return container.ExecCreateResponse{}, errdefs.NotFound(fmt.Errorf("Error: No such container: %s", ctn)) } From 23a1d4bd53ea009ab3976f5eb52c79e8a5554aa6 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Wed, 25 Feb 2026 15:58:19 -0600 Subject: [PATCH 59/63] add file size limits --- cmd/vela-worker/exec.go | 2 ++ cmd/vela-worker/flags.go | 14 +++++++---- cmd/vela-worker/run.go | 2 ++ executor/linux/linux.go | 5 ++++ executor/linux/opts.go | 24 +++++++++++++++++++ executor/linux/opts_test.go | 47 +++++++++++++++++++++++++++++++++++++ executor/linux/outputs.go | 13 ++++++++++ executor/setup.go | 6 +++++ executor/setup_test.go | 20 +++++++++------- 9 files changed, 121 insertions(+), 12 deletions(-) diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index 2113f5e1..f4f991b7 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -244,6 +244,8 @@ func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error Mock: w.Config.Mock, Driver: w.Config.Executor.Driver, MaxLogSize: w.Config.Executor.MaxLogSize, + FileSizeLimit: w.Config.Executor.FileSizeLimit, + BuildFileSizeLimit: w.Config.Executor.BuildFileSizeLimit, LogStreamingTimeout: w.Config.Executor.LogStreamingTimeout, EnforceTrustedRepos: w.Config.Executor.EnforceTrustedRepos, PrivilegedImages: w.Config.Runtime.PrivilegedImages, diff --git a/cmd/vela-worker/flags.go b/cmd/vela-worker/flags.go index eb7730d9..1a74c055 100644 --- a/cmd/vela-worker/flags.go +++ b/cmd/vela-worker/flags.go @@ -11,7 +11,6 @@ import ( "github.com/urfave/cli/v3" "github.com/go-vela/server/queue" - "github.com/go-vela/server/storage" "github.com/go-vela/worker/executor" "github.com/go-vela/worker/runtime" ) @@ -62,6 +61,16 @@ func flags() []cli.Flag { Sources: cli.EnvVars("WORKER_BUILD_TIMEOUT", "VELA_BUILD_TIMEOUT", "BUILD_TIMEOUT"), Value: 30 * time.Minute, }, + &cli.IntFlag{ + Name: "storage.file-size-limit", + Usage: "maximum file size (in MB) for a single file upload. 0 means no limit.", + Sources: cli.EnvVars("WORKER_STORAGE_FILE_SIZE_LIMIT", "VELA_STORAGE_FILE_SIZE_LIMIT", "STORAGE_FILE_SIZE_LIMIT"), + }, + &cli.IntFlag{ + Name: "storage.build-file-size-limit", + Usage: "maximum total size (in MB) for all file uploads in a single build. 0 means no limit.", + Sources: cli.EnvVars("WORKER_STORAGE_BUILD_FILE_SIZE_LIMIT", "VELA_STORAGE_BUILD_FILE_SIZE_LIMIT", "STORAGE_BUILD_FILE_SIZE_LIMIT"), + }, // Logger Flags @@ -121,8 +130,5 @@ func flags() []cli.Flag { f = append(f, runtime.Flags...) - // Storage Flags - f = append(f, storage.Flags...) - return f } diff --git a/cmd/vela-worker/run.go b/cmd/vela-worker/run.go index 6d1e419d..3d3c2e4e 100644 --- a/cmd/vela-worker/run.go +++ b/cmd/vela-worker/run.go @@ -105,6 +105,8 @@ func run(ctx context.Context, c *cli.Command) error { Executor: &executor.Setup{ Driver: c.String("executor.driver"), MaxLogSize: c.Uint("executor.max_log_size"), + FileSizeLimit: c.Int("storage.file-size-limit"), + BuildFileSizeLimit: c.Int("storage.build-file-size-limit"), LogStreamingTimeout: c.Duration("executor.log_streaming_timeout"), EnforceTrustedRepos: c.Bool("executor.enforce-trusted-repos"), OutputCtn: outputsCtn, diff --git a/executor/linux/linux.go b/executor/linux/linux.go index 83f54dcf..5c85ad3c 100644 --- a/executor/linux/linux.go +++ b/executor/linux/linux.go @@ -29,6 +29,7 @@ type ( Hostname string Version string OutputCtn *pipeline.Container + Uploaded int64 // clients for build actions secret *secretSvc @@ -37,6 +38,8 @@ type ( // private fields init *pipeline.Container maxLogSize uint + fileSizeLimit int64 + buildFileSizeLimit int64 logStreamingTimeout time.Duration privilegedImages []string enforceTrustedRepos bool @@ -74,6 +77,8 @@ func Equal(a, b *client) bool { a.Version == b.Version && reflect.DeepEqual(a.init, b.init) && a.maxLogSize == b.maxLogSize && + a.fileSizeLimit == b.fileSizeLimit && + a.buildFileSizeLimit == b.buildFileSizeLimit && reflect.DeepEqual(a.privilegedImages, b.privilegedImages) && a.enforceTrustedRepos == b.enforceTrustedRepos && reflect.DeepEqual(a.build, b.build) && diff --git a/executor/linux/opts.go b/executor/linux/opts.go index 5c394bae..469870d5 100644 --- a/executor/linux/opts.go +++ b/executor/linux/opts.go @@ -47,6 +47,30 @@ func WithMaxLogSize(size uint) Opt { } } +// WithFileSizeLimit sets the maximum file size (in MB) for a single file upload in the executor client for Linux. +func WithFileSizeLimit(limit int) Opt { + return func(c *client) error { + c.Logger.Trace("configuring file size limit in linux executor client") + + // set the file size limit in the client + c.fileSizeLimit = int64(limit) * 1024 * 1024 + + return nil + } +} + +// WithBuildFileSizeLimit sets the maximum total size (in MB) for all file uploads in a single build in the executor client for Linux. +func WithBuildFileSizeLimit(limit int) Opt { + return func(c *client) error { + c.Logger.Trace("configuring build file size limit in linux executor client") + + // set the build file size limit in the client + c.buildFileSizeLimit = int64(limit) * 1024 * 1024 + + return nil + } +} + // WithLogStreamingTimeout sets the log streaming timeout in the executor client for Linux. func WithLogStreamingTimeout(timeout time.Duration) Opt { return func(c *client) error { diff --git a/executor/linux/opts_test.go b/executor/linux/opts_test.go index 36196330..2f3f1a1a 100644 --- a/executor/linux/opts_test.go +++ b/executor/linux/opts_test.go @@ -70,6 +70,53 @@ func TestLinux_Opt_WithBuild(t *testing.T) { } func TestLinux_Opt_WithMaxLogSize(t *testing.T) { + // setup tests + tests := []struct { + name string + failure bool + fileSizeLimit int + buildFileSizeLimit int + }{ + { + name: "defined", + failure: false, + fileSizeLimit: 200, + buildFileSizeLimit: 1000, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _engine, err := New( + WithFileSizeLimit(test.fileSizeLimit), + WithBuildFileSizeLimit(test.buildFileSizeLimit), + ) + + if test.failure { + if err == nil { + t.Errorf("WithFileSizeLimit should have returned err") + } + + return // continue to next test + } + + if err != nil { + t.Errorf("WithFileSizeLimit returned err: %v", err) + } + + if !reflect.DeepEqual(_engine.fileSizeLimit, int64(test.fileSizeLimit*1024*1024)) { + t.Errorf("WithFileSizeLimit is %v, want %v", _engine.fileSizeLimit, int64(test.fileSizeLimit*1024*1024)) + } + + if !reflect.DeepEqual(_engine.buildFileSizeLimit, int64(test.buildFileSizeLimit*1024*1024)) { + t.Errorf("WithBuildFileSizeLimit is %v, want %v", _engine.buildFileSizeLimit, int64(test.buildFileSizeLimit*1024*1024)) + } + }) + } +} + +func TestLinux_Opt_WithFileSizeLimit(t *testing.T) { // setup tests tests := []struct { name string diff --git a/executor/linux/outputs.go b/executor/linux/outputs.go index 03a0d2aa..ce6a1522 100644 --- a/executor/linux/outputs.go +++ b/executor/linux/outputs.go @@ -265,6 +265,17 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, _ste return fmt.Errorf("unable to poll file content for %s: %w", filePath, err) } + // TODO: surface this skip to the user + if o.client.fileSizeLimit > 0 && size > o.client.fileSizeLimit { + logger.Infof("skipping file %s due to file size limit", filePath) + continue + } + + if o.client.buildFileSizeLimit > 0 && size+o.client.Uploaded > o.client.buildFileSizeLimit { + logger.Infof("skipping file %s due to build file size limit", filePath) + continue + } + // create storage object path objectName := fmt.Sprintf("%s/%s/%s/%s", b.GetRepo().GetOrg(), @@ -277,6 +288,8 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, _ste if err != nil { return fmt.Errorf("unable to upload object %s: %w", fileName, err) } + + o.client.Uploaded += size } return nil diff --git a/executor/setup.go b/executor/setup.go index cda031dd..a776e4ea 100644 --- a/executor/setup.go +++ b/executor/setup.go @@ -34,6 +34,10 @@ type Setup struct { Driver string // specifies the maximum log size MaxLogSize uint + // specifies file size limit for artifacts + FileSizeLimit int + // specifies build file size limit for artifacts + BuildFileSizeLimit int // specifies how long to wait after the build finishes // for log streaming to complete LogStreamingTimeout time.Duration @@ -79,6 +83,8 @@ func (s *Setup) Linux() (Engine, error) { opts := []linux.Opt{ linux.WithBuild(s.Build), linux.WithMaxLogSize(s.MaxLogSize), + linux.WithFileSizeLimit(s.FileSizeLimit), + linux.WithBuildFileSizeLimit(s.BuildFileSizeLimit), linux.WithLogStreamingTimeout(s.LogStreamingTimeout), linux.WithPrivilegedImages(s.PrivilegedImages), linux.WithEnforceTrustedRepos(s.EnforceTrustedRepos), diff --git a/executor/setup_test.go b/executor/setup_test.go index e89700a7..c48e7f26 100644 --- a/executor/setup_test.go +++ b/executor/setup_test.go @@ -72,6 +72,8 @@ func TestExecutor_Setup_Linux(t *testing.T) { want, err := linux.New( linux.WithBuild(_build), linux.WithMaxLogSize(2097152), + linux.WithFileSizeLimit(10), + linux.WithBuildFileSizeLimit(100), linux.WithLogStreamingTimeout(1*time.Second), linux.WithHostname("localhost"), linux.WithPipeline(_pipeline), @@ -84,14 +86,16 @@ func TestExecutor_Setup_Linux(t *testing.T) { } _setup := &Setup{ - Build: _build, - Client: _client, - Driver: constants.DriverLinux, - MaxLogSize: 2097152, - Hostname: "localhost", - Pipeline: _pipeline, - Runtime: _runtime, - Version: "v1.0.0", + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + MaxLogSize: 2097152, + FileSizeLimit: 10, + BuildFileSizeLimit: 100, + Hostname: "localhost", + Pipeline: _pipeline, + Runtime: _runtime, + Version: "v1.0.0", } // run test From c6519df19ad6539180a397ce9ac1efbe0be24b6a Mon Sep 17 00:00:00 2001 From: ecrupper Date: Thu, 26 Feb 2026 08:55:44 -0600 Subject: [PATCH 60/63] add default values --- cmd/vela-worker/flags.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/vela-worker/flags.go b/cmd/vela-worker/flags.go index 1a74c055..cfa460dc 100644 --- a/cmd/vela-worker/flags.go +++ b/cmd/vela-worker/flags.go @@ -65,11 +65,13 @@ func flags() []cli.Flag { Name: "storage.file-size-limit", Usage: "maximum file size (in MB) for a single file upload. 0 means no limit.", Sources: cli.EnvVars("WORKER_STORAGE_FILE_SIZE_LIMIT", "VELA_STORAGE_FILE_SIZE_LIMIT", "STORAGE_FILE_SIZE_LIMIT"), + Value: 100, }, &cli.IntFlag{ Name: "storage.build-file-size-limit", Usage: "maximum total size (in MB) for all file uploads in a single build. 0 means no limit.", Sources: cli.EnvVars("WORKER_STORAGE_BUILD_FILE_SIZE_LIMIT", "VELA_STORAGE_BUILD_FILE_SIZE_LIMIT", "STORAGE_BUILD_FILE_SIZE_LIMIT"), + Value: 500, }, // Logger Flags From 76e60815ea1968ed8681e6cf32746ea890491938 Mon Sep 17 00:00:00 2001 From: Tim Huynh Date: Thu, 26 Feb 2026 12:13:17 -0600 Subject: [PATCH 61/63] switch to using storage service to upload --- executor/linux/outputs.go | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/executor/linux/outputs.go b/executor/linux/outputs.go index ce6a1522..62ad4e67 100644 --- a/executor/linux/outputs.go +++ b/executor/linux/outputs.go @@ -12,12 +12,12 @@ import ( "strconv" envparse "github.com/hashicorp/go-envparse" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" "github.com/sirupsen/logrus" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/constants" + "github.com/go-vela/server/storage" ) // outputSvc handles communication with the outputs container during the build. @@ -228,14 +228,16 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, _ste return fmt.Errorf("unable to get sts storage creds %w with response code %d", err, res.StatusCode) } - stsStorageClient, err := minio.New(creds.Endpoint, - &minio.Options{ - Creds: credentials.NewStaticV4( - creds.AccessKey, - creds.SecretKey, - creds.SessionToken), - Secure: creds.Secure, - }) + stsStorageClient, err := storage.New(&storage.Setup{ + Enable: true, + Driver: constants.DriverMinio, + Endpoint: creds.Endpoint, + AccessKey: creds.AccessKey, + SecretKey: creds.SecretKey, + Bucket: creds.Bucket, + Secure: creds.Secure, + Token: creds.SessionToken, + }) if err != nil { return fmt.Errorf("unable to create sts storage client %w with response code %d", err, res.StatusCode) } @@ -283,8 +285,15 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, _ste strconv.FormatInt(b.GetNumber(), 10), fileName) + obj := &api.Object{ + Bucket: api.Bucket{ + BucketName: creds.Bucket, + }, + ObjectName: objectName, + FilePath: filePath, + } // upload file to storage - _, err = stsStorageClient.PutObject(ctx, creds.Bucket, objectName, reader, size, minio.PutObjectOptions{}) + err = stsStorageClient.UploadObject(ctx, obj, reader, size) if err != nil { return fmt.Errorf("unable to upload object %s: %w", fileName, err) } From 73867ade7c4611cd867c82f753441b52a26b6652 Mon Sep 17 00:00:00 2001 From: Tim Huynh Date: Thu, 26 Feb 2026 13:37:39 -0600 Subject: [PATCH 62/63] use creds values --- executor/linux/outputs.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/executor/linux/outputs.go b/executor/linux/outputs.go index 62ad4e67..cf34f632 100644 --- a/executor/linux/outputs.go +++ b/executor/linux/outputs.go @@ -16,7 +16,6 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" - "github.com/go-vela/server/constants" "github.com/go-vela/server/storage" ) @@ -229,8 +228,8 @@ func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, _ste } stsStorageClient, err := storage.New(&storage.Setup{ - Enable: true, - Driver: constants.DriverMinio, + Enable: creds.Enable, + Driver: creds.Driver, Endpoint: creds.Endpoint, AccessKey: creds.AccessKey, SecretKey: creds.SecretKey, From 37114702f44f6641252fc2c6e9bef2d4e8664389 Mon Sep 17 00:00:00 2001 From: Tim Huynh Date: Thu, 26 Feb 2026 15:46:37 -0600 Subject: [PATCH 63/63] update to rc2 --- go.mod | 8 ++++---- go.sum | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 04435e5a..acccf47b 100644 --- a/go.mod +++ b/go.mod @@ -9,12 +9,11 @@ require ( github.com/docker/docker v28.5.2+incompatible github.com/docker/go-units v0.5.0 github.com/gin-gonic/gin v1.11.0 - github.com/go-vela/sdk-go v0.28.0-rc1 - github.com/go-vela/server v0.28.0-rc1 + github.com/go-vela/sdk-go v0.28.0-rc2 + github.com/go-vela/server v0.28.0-rc2 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/go-cmp v0.7.0 github.com/joho/godotenv v1.5.1 - github.com/minio/minio-go/v7 v7.0.98 github.com/opencontainers/image-spec v1.1.1 github.com/prometheus/client_golang v1.23.2 github.com/sirupsen/logrus v1.9.4 @@ -62,6 +61,7 @@ require ( github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.98 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -98,7 +98,7 @@ require ( github.com/Masterminds/sprig/v3 v3.3.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/PuerkitoBio/purell v1.2.1 // indirect - github.com/alicebob/miniredis/v2 v2.36.1 // indirect + github.com/alicebob/miniredis/v2 v2.37.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect diff --git a/go.sum b/go.sum index 288d5367..9bce57a8 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,8 @@ github.com/PuerkitoBio/purell v1.2.1/go.mod h1:ZwHcC/82TOaovDi//J/804umJFFmbOHPn github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/adhocore/gronx v1.19.6 h1:5KNVcoR9ACgL9HhEqCm5QXsab/gI4QDIybTAWcXDKDc= github.com/adhocore/gronx v1.19.6/go.mod h1:7oUY1WAU8rEJWmAxXR2DN0JaO4gi9khSgKjiRypqteg= -github.com/alicebob/miniredis/v2 v2.36.1 h1:Dvc5oAnNOr7BIfPn7tF269U8DvRW1dBG2D5n0WrfYMI= -github.com/alicebob/miniredis/v2 v2.36.1/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= +github.com/alicebob/miniredis/v2 v2.37.0 h1:RheObYW32G1aiJIj81XVt78ZHJpHonHLHW7OLIshq68= +github.com/alicebob/miniredis/v2 v2.37.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= @@ -123,10 +123,10 @@ github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy0 github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/go-vela/sdk-go v0.28.0-rc1 h1:fNWM6t6WCUt3P6sqg7k98ls1zVlKkRs4QggxIwlj2rM= -github.com/go-vela/sdk-go v0.28.0-rc1/go.mod h1:XSs3YyiqD0Vtni8KXAgLZYTOiUK1mhNZVYAg+LCh8nU= -github.com/go-vela/server v0.28.0-rc1 h1:GkQE6i+CgvQsjK5TNScfP/EnbarfGPTLA+m/Pu268nQ= -github.com/go-vela/server v0.28.0-rc1/go.mod h1:pbQZTnaLzJelBfC1b9YfB/DnkMp90ICN7nh0Gy+mq4I= +github.com/go-vela/sdk-go v0.28.0-rc2 h1:givDWdK392Q+fbfIoa9SVbJSnHdHozVSc/TTww23J6w= +github.com/go-vela/sdk-go v0.28.0-rc2/go.mod h1:hzYyyf77s43fpuuCQ/CKtlCGkWjqAYXWftQH95qwE4M= +github.com/go-vela/server v0.28.0-rc2 h1:BoAcEalEe6t1h57X+fgJQitVrc7K8fOM7Akr6iCsuy8= +github.com/go-vela/server v0.28.0-rc2/go.mod h1:2enRAFzhb7hVUXaj7b9v1uuVHImbGktJ6SpkcMsWdN0= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=