Artifact plugins

.features/pending/artifact-plugins.md

@@ -0,0 +1,8 @@
+Description: Artifact Drivers as plugins
+Author: [Alan Clucas](https://github.com/Joibel), [JP Zivalich](https://github.com/JPZ13), [Elliot Gunton](https://github.com/elliotgunton)
+Component: General
+Issues: 5862
+
+Artifact Drivers can now be added via a plugin mechanism.
+You can write a GRPC server which acts as an artifact driver to upload and download artifacts to a repository, and supply that as a docker image.
+Argo workflows can then use that as a driver.

.spelling

@@ -96,6 +96,7 @@ MySQL
 Nagal
 Nano
 Nginx
+Node.JS
 Node.JS.
 OAuth
 OAuth2
@@ -272,6 +273,7 @@ v3.6.1
 v3.6.5
 v3.7
 v3.7.0
+v4.0
 validator
 vendored
 versioned

Makefile

@@ -223,7 +223,7 @@ define protoc
       --grpc-gateway_out=logtostderr=true:$(GOPATH)/src \
       --swagger_out=logtostderr=true,fqn_for_swagger_name=true:. \
       $(1)
-     perl -i -pe 's|argoproj/argo-workflows/|argoproj/argo-workflows/v3/|g' `echo "$(1)" | sed 's/proto/pb.go/g'`
+    perl -i -pe 's|argoproj/argo-workflows/|argoproj/argo-workflows/v3/|g' `echo "$(1)" | sed 's/proto/pb.go/g'`
 
 endef
 
@@ -446,6 +446,8 @@ pkg/apis/workflow/v1alpha1/generated.proto: $(TOOL_GO_TO_PROTOBUF) $(PROTO_BINAR
 	[ -e ./v3 ] || ln -s . v3
 	# Format proto files. Formatting changes generated code, so we do it here, rather that at lint time.
 	# Why clang-format? Google uses it.
+	@echo "*** This will fail if your code has compilation errors, without reporting those as the cause."
+	@echo "*** So fix them first."
 	find pkg/apiclient -name '*.proto'|xargs clang-format -i
 	$(TOOL_GO_TO_PROTOBUF) \
 		--go-header-file=./hack/custom-boilerplate.go.txt \
@@ -454,7 +456,7 @@ pkg/apis/workflow/v1alpha1/generated.proto: $(TOOL_GO_TO_PROTOBUF) $(PROTO_BINAR
 		--proto-import $(GOPATH)/src
 	# Delete the link
 	[ -e ./v3 ] && rm -rf v3
-	touch pkg/apis/workflow/v1alpha1/generated.proto
+	touch $@
 
 # this target will also create a .pb.go and a .pb.gw.go file, but in Make 3 we cannot use _grouped target_, instead we must choose
 # on file to represent all of them
@@ -937,3 +939,10 @@ devcontainer-build: $(TOOL_DEVCONTAINER)
 .PHONY: devcontainer-up
 devcontainer-up: $(TOOL_DEVCONTAINER)
 	devcontainer up --workspace-folder .
+
+# gRPC/protobuf generation for artifact.proto
+pkg/apiclient/artifact/artifact.swagger.json: $(PROTO_BINARIES) $(TYPES) pkg/apiclient/artifact/artifact.proto
+	$(call protoc,pkg/apiclient/artifact/artifact.proto)
+
+# Add artifact-proto to swagger dependencies
+swagger: pkg/apiclient/artifact/artifact.swagger.json

README.md

@@ -87,7 +87,7 @@ Check out our [Java, Golang and Python clients](docs/client-libraries.md).
 An incomplete list of features Argo Workflows provide:
 
 * UI to visualize and manage Workflows
-* Artifact support (S3, Artifactory, Alibaba Cloud OSS, Azure Blob Storage, HTTP, Git, GCS, raw)
+* Artifact support (S3, Artifactory, Alibaba Cloud OSS, Azure Blob Storage, HTTP, Git, GCS, raw, plugins)
 * Workflow templating to store commonly used Workflows in the cluster
 * Archiving Workflows after executing for later access
 * Scheduled workflows using cron

cmd/argoexec/commands/artifact/delete.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/spf13/cobra"
 
@@ -16,10 +17,13 @@ import (
 	"github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
 	workflow "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned"
 	wfv1alpha1 "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
+	"github.com/argoproj/argo-workflows/v3/util/logging"
 	"github.com/argoproj/argo-workflows/v3/util/retry"
 	waitutil "github.com/argoproj/argo-workflows/v3/util/wait"
-	executor "github.com/argoproj/argo-workflows/v3/workflow/artifacts"
+	"github.com/argoproj/argo-workflows/v3/workflow/artifacts"
 	"github.com/argoproj/argo-workflows/v3/workflow/common"
+	"github.com/argoproj/argo-workflows/v3/workflow/executor"
+	"github.com/argoproj/argo-workflows/v3/workflow/executor/emissary"
 )
 
 func NewArtifactDeleteCommand() *cobra.Command {
@@ -28,6 +32,7 @@ func NewArtifactDeleteCommand() *cobra.Command {
 		SilenceUsage: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
+			logger := logging.RequireLoggerFromContext(ctx)
 			namespace := client.Namespace(ctx)
 			clientConfig := client.GetConfig()
 
@@ -46,7 +51,25 @@ func NewArtifactDeleteCommand() *cobra.Command {
 				if err != nil {
 					return err
 				}
+				logger.Info(ctx, "artifacts deleted")
 			}
+			em, err := emissary.New()
+			if err != nil {
+				return err
+			}
+			pluginNamesEnv := os.Getenv(common.EnvVarArtifactPluginNames)
+			if pluginNamesEnv == "" {
+				logger.Info(ctx, "no artifact sidecars to kill")
+				return nil
+			}
+			artifactSidecars := strings.Split(pluginNamesEnv, ",")
+			logger.WithFields(logging.Fields{"artifactSidecars": artifactSidecars}).Info(ctx, "killing artifact sidecars")
+			err = em.Kill(ctx, artifactSidecars, executor.GetTerminationGracePeriodDuration())
+			if err != nil {
+				logger.WithError(err).WithFields(logging.Fields{"artifactSidecars": artifactSidecars}).Error(ctx, "failed to kill artifact sidecars")
+				return err
+			}
+			logger.WithFields(logging.Fields{"artifactSidecars": artifactSidecars}).Info(ctx, "artifact sidecars killed")
 			return nil
 		},
 	}
@@ -79,7 +102,7 @@ func deleteArtifacts(labelSelector string, ctx context.Context, artifactGCTaskIn
 					}
 				}
 
-				drv, err := executor.NewDriver(ctx, &artifact, resources)
+				drv, err := artifacts.NewDriver(ctx, &artifact, resources)
 				if err != nil {
 					return err
 				}