From d54f706aa73b2f1bd21e70b5953eaa170981c708 Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Tue, 11 Aug 2020 14:03:28 -0400 Subject: [PATCH 01/18] implement cloud build config support (part one) --- cmd/root.go | 17 +++- go.mod | 1 + go.sum | 5 + internal/cmd/cmd.go | 88 +++++++++++++++++ internal/lifecycle/cloud_build_config.go | 116 +++++++++++++++++++++++ internal/lifecycle/lifecycle.go | 22 ++++- internal/sample/sample.go | 4 +- internal/util/tempfiles.go | 31 ++++++ main.go | 5 +- 9 files changed, 279 insertions(+), 10 deletions(-) create mode 100644 internal/cmd/cmd.go create mode 100644 internal/lifecycle/cloud_build_config.go create mode 100644 internal/util/tempfiles.go diff --git a/cmd/root.go b/cmd/root.go index c58d09d..94ae4fb 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -40,11 +40,7 @@ var ( } log.Println("Setting up configuration values") - // Set up config file location - viper.SetConfigName("config") - viper.SetConfigType("yaml") - viper.AddConfigPath(sampleDir) - s, err := sample.NewSample(sampleDir) + s, err := sample.NewSample(sampleDir, viper.GetStringMapString("cloud_build_subs")) if err != nil { return err } @@ -96,4 +92,15 @@ func Execute() error { // init initializes the tool. func init() { // Initialization goes here + // Set up config file location + viper.SetConfigName("config") + viper.SetConfigType("yaml") + //viper.AddConfigPath(sampleDir) + + viper.SetEnvPrefix("sst") + + viper.SetDefault("cloud_build_subs", map[string]string{}) + viper.BindEnv("cloud_build_subs") + rootCmd.PersistentFlags().StringToString("cloud-build-subs", map[string]string{}, "") + viper.BindPFlag("cloud_build_subs", rootCmd.PersistentFlags().Lookup("cloud-build-subs")) } diff --git a/go.mod b/go.mod index e1963eb..b264c3e 100644 --- a/go.mod +++ b/go.mod @@ -6,4 +6,5 @@ require ( github.com/getkin/kin-openapi v0.18.0 github.com/spf13/cobra v1.0.0 github.com/spf13/viper v1.7.1 + gopkg.in/yaml.v2 v2.3.0 ) diff --git a/go.sum b/go.sum index 9b799b3..8829327 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,7 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -71,6 +72,7 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -103,6 +105,7 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -159,7 +162,9 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go new file mode 100644 index 0000000..a40c005 --- /dev/null +++ b/internal/cmd/cmd.go @@ -0,0 +1,88 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "github.com/GoogleCloudPlatform/serverless-sample-tester/internal/sample" + "github.com/GoogleCloudPlatform/serverless-sample-tester/internal/util" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "log" + "os/exec" + "path/filepath" +) + +func InitConfig(rootCmd *cobra.Command) { + viper.SetEnvPrefix("sst") + + viper.BindEnv("substitutions") + rootCmd.PersistentFlags().StringToString("substitutions", map[string]string{}, "dsfasd") + viper.BindPFlag("substitutions", rootCmd.PersistentFlags().Lookup("substitutions")) +} + +// Root is responsible for the root command. It handles the application flow. +func Root(cmd *cobra.Command, args []string) error { + // Parse sample directory from command line argument + sampleDir, err := filepath.Abs(filepath.Dir(args[0])) + if err != nil { + return err + } + + log.Println("Setting up configuration values") + s, err := sample.NewSample(sampleDir, viper.GetStringMapString("substitutions")) + if err != nil { + return err + } + + log.Println("Loading test endpoints") + swagger := util.LoadTestEndpoints() + + log.Println("Building and deploying sample to Cloud Run") + err = s.BuildDeployLifecycle.Execute(s.Dir) + defer s.Service.Delete(s.Dir) + defer s.DeleteCloudContainerImage() + if err != nil { + return fmt.Errorf("[cmd.Root] building and deploying sample to Cloud Run: %w", err) + } + + log.Println("Getting identity token for gcloud auhtorized account") + var identToken string + + a := append(util.GcloudCommonFlags, "auth", "print-identity-token") + identToken, err = util.ExecCommand(exec.Command("gcloud", a...), s.Dir) + + if err != nil { + return fmt.Errorf("[cmd.Root] getting identity token for gcloud auhtorized account: %w", err) + } + + log.Println("Checking endpoints for expected results") + serviceURL, err := s.Service.URL(s.Dir) + if err != nil { + return fmt.Errorf("[cmd.Root] getting Cloud Run service URL: %w", err) + } + + log.Println("Validating Cloud Run service endpoints for expected status codes") + allTestsPassed, err := util.ValidateEndpoints(serviceURL, &swagger.Paths, identToken) + if err != nil { + return fmt.Errorf("[cmd.Root] validating Cloud Run service endpoints for expected status codes: %w", err) + } + + if !allTestsPassed { + return fmt.Errorf("all tests did not pass") + } + + return nil +} diff --git a/internal/lifecycle/cloud_build_config.go b/internal/lifecycle/cloud_build_config.go new file mode 100644 index 0000000..0156e5a --- /dev/null +++ b/internal/lifecycle/cloud_build_config.go @@ -0,0 +1,116 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lifecycle + +import ( + "fmt" + "github.com/GoogleCloudPlatform/serverless-sample-tester/internal/util" + "gopkg.in/yaml.v2" + "io/ioutil" + "os/exec" + "regexp" + "strings" +) + +func parseCloudBuildConfig(filename, serviceName, gcrURL string, substitutions map[string]string) (Lifecycle, error) { + config := make(map[string]interface{}) + + buildConfigBytes, err := ioutil.ReadFile(filename) + if err != nil { + return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] reading Cloud Build config file: %w", err) + } + + err = yaml.Unmarshal(buildConfigBytes, &config) + if err != nil { + return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] unmarshaling Cloud Build config file: %w", err) + } + + + // Replace Cloud Run service names and Cloud Container Registry URLs + for stepIndex := range config["steps"].([]interface{}) { + runCommand := false + lastArgIndex := -1 + + for argIndex := range config["steps"].([]interface{})[stepIndex].(map[interface{}]interface{})["args"].([]interface{}) { + arg := config["steps"].([]interface{})[stepIndex].(map[interface{}]interface{})["args"].([]interface{})[argIndex].(string) + + if strings.Contains(arg, "run") { + runCommand = true + } + + if !strings.Contains(arg, "--") { + lastArgIndex = argIndex + } + + arg = replaceGCRURL(arg, gcrURL) + config["steps"].([]interface{})[stepIndex].(map[interface{}]interface{})["args"].([]interface{})[argIndex] = arg + } + + if runCommand && lastArgIndex != -1 { + config["steps"].([]interface{})[stepIndex].(map[interface{}]interface{})["args"].([]interface{})[lastArgIndex] = serviceName + } + } + + configMarshalBytes, err := yaml.Marshal(&config) + if err != nil { + return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] marshaling modified Cloud Build config: %w", err) + } + + tempBuildConfigFile, err := util.CreateTempFile() + if err != nil { + return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] creating temporary file: %w", err) + } + + if _, err := tempBuildConfigFile.Write(configMarshalBytes); err != nil { + return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] writing to temporary file: %w", err) + } + if err := tempBuildConfigFile.Close(); err != nil { + return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] closing temporary file: %w", err) + } + + return buildCloudBuildConfigLifecycle(tempBuildConfigFile.Name(), substitutions), nil +} + +func buildCloudBuildConfigLifecycle(buildConfigFilename string, substitutions map[string]string) Lifecycle { + a := append(util.GcloudCommonFlags, "builds", "submit", + fmt.Sprintf("--config=%s", buildConfigFilename)) + + subsitutions, empty := substitutionsString(substitutions) + if !empty { + a = append(a, fmt.Sprintf("--substitutions=%s", subsitutions)) + } + + return Lifecycle{exec.Command("gcloud", a...)} +} + +// replaceServiceName takes a terminal command string as input and replaces the URL of a container image stored in the +// GCP Container Registry with the given URL. +func replaceGCRURL(commandStr string, gcrURL string) string { + re := regexp.MustCompile(`gcr.io/.+/\S+`) + return re.ReplaceAllString(commandStr, gcrURL) +} + +func substitutionsString(m map[string]string) (string, bool) { + if len(m) == 0 { + return "", true + } + + var subs []string + for k, v := range m { + subs = append(subs, fmt.Sprintf("%s=%s", k, v)) + } + + return strings.Join(subs, ","), false +} diff --git a/internal/lifecycle/lifecycle.go b/internal/lifecycle/lifecycle.go index 9aa8801..63b2652 100644 --- a/internal/lifecycle/lifecycle.go +++ b/internal/lifecycle/lifecycle.go @@ -47,7 +47,24 @@ func (l Lifecycle) Execute(commandsDir string) error { // NewLifecycle tries to parse the different options provided for build and deploy command configuration. If none of // those options are set up, it falls back to reasonable defaults based on whether the sample is java-based // (has a pom.xml) that doesn't have a Dockerfile or isn't. -func NewLifecycle(sampleDir, serviceName, gcrURL string) (Lifecycle, error) { +func NewLifecycle(sampleDir, serviceName, gcrURL string, cloudBuildConfSubs map[string]string) (Lifecycle, error) { + // First try Cloud Build Config file + cloudBuildConfigPath := fmt.Sprintf("%s/cloudbuild.yaml", sampleDir) + + _, err := os.Stat(cloudBuildConfigPath) + cloudBuildConfigE := err == nil + + if cloudBuildConfigE { + lifecycle, err := parseCloudBuildConfig(cloudBuildConfigPath, serviceName, gcrURL, cloudBuildConfSubs) + if err == nil { + log.Println("Using cloud build config file") + return lifecycle, nil + } + + return nil, fmt.Errorf("[lifecycle.NewLifecycle] using cloud build config file %s: %v\n", cloudBuildConfigPath, err) + } + + // Then try README parsing var readmePath string // Searching for config file if err := viper.ReadInConfig(); err == nil { @@ -76,10 +93,11 @@ func NewLifecycle(sampleDir, serviceName, gcrURL string) (Lifecycle, error) { log.Println("No README.md found") } + // Finally fall back to reasonable defaults pomPath := filepath.Join(sampleDir, "pom.xml") dockerfilePath := filepath.Join(sampleDir, "Dockerfile") - _, err := os.Stat(pomPath) + _, err = os.Stat(pomPath) pomE := err == nil _, err = os.Stat(dockerfilePath) diff --git a/internal/sample/sample.go b/internal/sample/sample.go index 29d1f09..5e34ba0 100644 --- a/internal/sample/sample.go +++ b/internal/sample/sample.go @@ -44,7 +44,7 @@ type Sample struct { } // NewSample creates a new sample object for the sample located in the provided local directory. -func NewSample(dir string) (*Sample, error) { +func NewSample(dir string, cloudBuildConfSubs map[string]string) (*Sample, error) { name := sampleName(dir) containerTag, err := cloudContainerImageTag(name, dir) @@ -66,7 +66,7 @@ func NewSample(dir string) (*Sample, error) { } service := gcloud.CloudRunService{Name: serviceName} - buildDeployLifecycle, err := lifecycle.NewLifecycle(dir, service.Name, cloudContainerImageURL) + buildDeployLifecycle, err := lifecycle.NewLifecycle(dir, service.Name, cloudContainerImageURL, cloudBuildConfSubs) if err != nil { return nil, fmt.Errorf("lifecycle.NewLifecycle: %w", err) } diff --git a/internal/util/tempfiles.go b/internal/util/tempfiles.go new file mode 100644 index 0000000..0691ea7 --- /dev/null +++ b/internal/util/tempfiles.go @@ -0,0 +1,31 @@ +package util + +import ( + "fmt" + "io/ioutil" + "log" + "os" +) + +var tempFiles []*os.File + +func CreateTempFile() (*os.File, error) { + tempFile, err := ioutil.TempFile("", "example") + if err != nil { + return tempFile, fmt.Errorf("[util.CreateTempFile] creating temp file: %w\n", err) + } + + tempFiles = append(tempFiles, tempFile) + return tempFile, nil +} + +func RemoveTempFiles() error { + for _, tempFile := range tempFiles { + err := os.Remove(tempFile.Name()) + if err != nil { + log.Printf("Error removing Temp File: %v\n", err) + } + } + + return nil +} diff --git a/main.go b/main.go index ee2d21a..ee123d2 100644 --- a/main.go +++ b/main.go @@ -16,8 +16,11 @@ package main import ( "github.com/GoogleCloudPlatform/serverless-sample-tester/cmd" + "log" ) func main() { - cmd.Execute() + if err := cmd.Execute(); err != nil { + log.Fatalf("Error ocurred in the execution of this program: %v\n", err) + } } From 1dfb7230ac1470fa908bcfea7b544b8e73424bdb Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Mon, 17 Aug 2020 17:01:41 -0400 Subject: [PATCH 02/18] add region substitution for cloud build config --- internal/lifecycle/cloud_build_config.go | 20 ++++++++++---------- internal/lifecycle/lifecycle.go | 4 ++-- internal/sample/sample.go | 13 ++++++++++++- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/internal/lifecycle/cloud_build_config.go b/internal/lifecycle/cloud_build_config.go index 0156e5a..fea4e87 100644 --- a/internal/lifecycle/cloud_build_config.go +++ b/internal/lifecycle/cloud_build_config.go @@ -24,7 +24,9 @@ import ( "strings" ) -func parseCloudBuildConfig(filename, serviceName, gcrURL string, substitutions map[string]string) (Lifecycle, error) { +const runRegionSubstitution = "_SST_RUN_REGION" + +func parseCloudBuildConfig(filename, serviceName, gcrURL, runRegion string, substitutions map[string]string) (Lifecycle, error) { config := make(map[string]interface{}) buildConfigBytes, err := ioutil.ReadFile(filename) @@ -37,7 +39,6 @@ func parseCloudBuildConfig(filename, serviceName, gcrURL string, substitutions m return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] unmarshaling Cloud Build config file: %w", err) } - // Replace Cloud Run service names and Cloud Container Registry URLs for stepIndex := range config["steps"].([]interface{}) { runCommand := false @@ -80,14 +81,14 @@ func parseCloudBuildConfig(filename, serviceName, gcrURL string, substitutions m return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] closing temporary file: %w", err) } - return buildCloudBuildConfigLifecycle(tempBuildConfigFile.Name(), substitutions), nil + return buildCloudBuildConfigLifecycle(tempBuildConfigFile.Name(), runRegion, substitutions), nil } -func buildCloudBuildConfigLifecycle(buildConfigFilename string, substitutions map[string]string) Lifecycle { +func buildCloudBuildConfigLifecycle(buildConfigFilename, runRegion string, substitutions map[string]string) Lifecycle { a := append(util.GcloudCommonFlags, "builds", "submit", fmt.Sprintf("--config=%s", buildConfigFilename)) - subsitutions, empty := substitutionsString(substitutions) + subsitutions, empty := substitutionsString(substitutions, runRegion) if !empty { a = append(a, fmt.Sprintf("--substitutions=%s", subsitutions)) } @@ -102,12 +103,11 @@ func replaceGCRURL(commandStr string, gcrURL string) string { return re.ReplaceAllString(commandStr, gcrURL) } -func substitutionsString(m map[string]string) (string, bool) { - if len(m) == 0 { - return "", true - } - +// substitutionsString +func substitutionsString(m map[string]string, runRegion string) (string, bool) { var subs []string + subs = append(subs, fmt.Sprintf("%s=%s", runRegionSubstitution, runRegion)) + for k, v := range m { subs = append(subs, fmt.Sprintf("%s=%s", k, v)) } diff --git a/internal/lifecycle/lifecycle.go b/internal/lifecycle/lifecycle.go index 63b2652..f3e5f20 100644 --- a/internal/lifecycle/lifecycle.go +++ b/internal/lifecycle/lifecycle.go @@ -47,7 +47,7 @@ func (l Lifecycle) Execute(commandsDir string) error { // NewLifecycle tries to parse the different options provided for build and deploy command configuration. If none of // those options are set up, it falls back to reasonable defaults based on whether the sample is java-based // (has a pom.xml) that doesn't have a Dockerfile or isn't. -func NewLifecycle(sampleDir, serviceName, gcrURL string, cloudBuildConfSubs map[string]string) (Lifecycle, error) { +func NewLifecycle(sampleDir, serviceName, gcrURL, runRegion string, cloudBuildConfSubs map[string]string) (Lifecycle, error) { // First try Cloud Build Config file cloudBuildConfigPath := fmt.Sprintf("%s/cloudbuild.yaml", sampleDir) @@ -55,7 +55,7 @@ func NewLifecycle(sampleDir, serviceName, gcrURL string, cloudBuildConfSubs map[ cloudBuildConfigE := err == nil if cloudBuildConfigE { - lifecycle, err := parseCloudBuildConfig(cloudBuildConfigPath, serviceName, gcrURL, cloudBuildConfSubs) + lifecycle, err := parseCloudBuildConfig(cloudBuildConfigPath, serviceName, gcrURL, runRegion, cloudBuildConfSubs) if err == nil { log.Println("Using cloud build config file") return lifecycle, nil diff --git a/internal/sample/sample.go b/internal/sample/sample.go index 5e34ba0..a56de6d 100644 --- a/internal/sample/sample.go +++ b/internal/sample/sample.go @@ -41,6 +41,9 @@ type Sample struct { // The URL location of this sample's build container image in the GCP Container Registry. cloudContainerImageURL string + + // The Cloud Run region this sample will deploy to. + runRegion string } // NewSample creates a new sample object for the sample located in the provided local directory. @@ -60,13 +63,20 @@ func NewSample(dir string, cloudBuildConfSubs map[string]string) (*Sample, error } cloudContainerImageURL := fmt.Sprintf("gcr.io/%s/%s", projectID, containerTag) + a = append(util.GcloudCommonFlags, "config", "get-value", "run/region") + runRegion, err := util.ExecCommand(exec.Command("gcloud", a...), dir) + + if err != nil { + return nil, fmt.Errorf("[sample.NewSample] getting gcloud cloud run default region: %w", err) + } + serviceName, err := gcloud.ServiceName(name) if err != nil { return nil, fmt.Errorf("gcloud.ServiceName: %s sample: %w", name, err) } service := gcloud.CloudRunService{Name: serviceName} - buildDeployLifecycle, err := lifecycle.NewLifecycle(dir, service.Name, cloudContainerImageURL, cloudBuildConfSubs) + buildDeployLifecycle, err := lifecycle.NewLifecycle(dir, service.Name, cloudContainerImageURL, runRegion, cloudBuildConfSubs) if err != nil { return nil, fmt.Errorf("lifecycle.NewLifecycle: %w", err) } @@ -77,6 +87,7 @@ func NewSample(dir string, cloudBuildConfSubs map[string]string) (*Sample, error Service: service, BuildDeployLifecycle: buildDeployLifecycle, cloudContainerImageURL: cloudContainerImageURL, + runRegion: runRegion, } return s, nil } From a3fcdf62e1c36a9f0331986de0ff1d9a91bbfa67 Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Mon, 17 Aug 2020 19:47:02 -0400 Subject: [PATCH 03/18] refactor cloud build config implementation --- go.mod | 1 + internal/lifecycle/cloud_build_config.go | 58 +++++++++++------------- internal/lifecycle/lifecycle.go | 41 ++++++++++++++++- internal/lifecycle/readme.go | 49 +++----------------- 4 files changed, 73 insertions(+), 76 deletions(-) diff --git a/go.mod b/go.mod index b264c3e..9008205 100644 --- a/go.mod +++ b/go.mod @@ -6,5 +6,6 @@ require ( github.com/getkin/kin-openapi v0.18.0 github.com/spf13/cobra v1.0.0 github.com/spf13/viper v1.7.1 + github.com/spf13/pflag v1.0.3 gopkg.in/yaml.v2 v2.3.0 ) diff --git a/internal/lifecycle/cloud_build_config.go b/internal/lifecycle/cloud_build_config.go index fea4e87..1522f23 100644 --- a/internal/lifecycle/cloud_build_config.go +++ b/internal/lifecycle/cloud_build_config.go @@ -20,13 +20,17 @@ import ( "gopkg.in/yaml.v2" "io/ioutil" "os/exec" - "regexp" "strings" ) +// TODO: comment here const runRegionSubstitution = "_SST_RUN_REGION" -func parseCloudBuildConfig(filename, serviceName, gcrURL, runRegion string, substitutions map[string]string) (Lifecycle, error) { +// getCloudBuildConfigLifecycle returns a Lifecycle for the executing the provided Cloud Build config file. It creates +// and uses a temporary copy of the file where it replaces the Cloud Run service names and Container Registry tags with +// the provided inputs. It provides also passes in the provided substitutions as well a runRegionSubstitution with the +// provided region. +func getCloudBuildConfigLifecycle(filename, serviceName, gcrURL, runRegion string, substitutions map[string]string) (Lifecycle, error) { config := make(map[string]interface{}) buildConfigBytes, err := ioutil.ReadFile(filename) @@ -39,29 +43,23 @@ func parseCloudBuildConfig(filename, serviceName, gcrURL, runRegion string, subs return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] unmarshaling Cloud Build config file: %w", err) } - // Replace Cloud Run service names and Cloud Container Registry URLs + // Replace Cloud Run service names and Container Registry URLs for stepIndex := range config["steps"].([]interface{}) { - runCommand := false - lastArgIndex := -1 - + var args []string for argIndex := range config["steps"].([]interface{})[stepIndex].(map[interface{}]interface{})["args"].([]interface{}) { arg := config["steps"].([]interface{})[stepIndex].(map[interface{}]interface{})["args"].([]interface{})[argIndex].(string) + arg = gcrURLRegexp.ReplaceAllString(arg, gcrURL) - if strings.Contains(arg, "run") { - runCommand = true - } - - if !strings.Contains(arg, "--") { - lastArgIndex = argIndex - } - - arg = replaceGCRURL(arg, gcrURL) - config["steps"].([]interface{})[stepIndex].(map[interface{}]interface{})["args"].([]interface{})[argIndex] = arg + args = append(args, arg) } - if runCommand && lastArgIndex != -1 { - config["steps"].([]interface{})[stepIndex].(map[interface{}]interface{})["args"].([]interface{})[lastArgIndex] = serviceName + prog := config["steps"].([]interface{})[stepIndex].(map[interface{}]interface{})["name"].(string) + err := replaceServiceName(prog, args, serviceName) + if err != nil { + return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] replacing Cloud Run service name in Cloud Build config step args: %w", err) } + + config["steps"].([]interface{})[stepIndex].(map[interface{}]interface{})["args"] = args } configMarshalBytes, err := yaml.Marshal(&config) @@ -84,27 +82,23 @@ func parseCloudBuildConfig(filename, serviceName, gcrURL, runRegion string, subs return buildCloudBuildConfigLifecycle(tempBuildConfigFile.Name(), runRegion, substitutions), nil } +// buildCloudBuildConfigLifecycle returns a Lifecycle with a single command that calls gcloud builds subit and passes +// in the provided Cloud Build config file. It also adds a `--substitutions` flag according to the substitutions +// provided and adds a substitution for the Cloud Run region with the name runRegionSubstitution and value provided. func buildCloudBuildConfigLifecycle(buildConfigFilename, runRegion string, substitutions map[string]string) Lifecycle { a := append(util.GcloudCommonFlags, "builds", "submit", fmt.Sprintf("--config=%s", buildConfigFilename)) - subsitutions, empty := substitutionsString(substitutions, runRegion) - if !empty { - a = append(a, fmt.Sprintf("--substitutions=%s", subsitutions)) - } + subsitutions := substitutionsString(substitutions, runRegion) + a = append(a, fmt.Sprintf("--substitutions=%s", subsitutions)) return Lifecycle{exec.Command("gcloud", a...)} } -// replaceServiceName takes a terminal command string as input and replaces the URL of a container image stored in the -// GCP Container Registry with the given URL. -func replaceGCRURL(commandStr string, gcrURL string) string { - re := regexp.MustCompile(`gcr.io/.+/\S+`) - return re.ReplaceAllString(commandStr, gcrURL) -} - -// substitutionsString -func substitutionsString(m map[string]string, runRegion string) (string, bool) { +// substitutionsString takes a string to string map and converts it into an argument for the `gcloud builds submit` +// `--config` file. It treats the keys in the map as the substitutions and the values as the substitution values. It +// also adds a substitution for the Cloud Run region with the name runRegionSubstitution and value provided. +func substitutionsString(m map[string]string, runRegion string) string { var subs []string subs = append(subs, fmt.Sprintf("%s=%s", runRegionSubstitution, runRegion)) @@ -112,5 +106,5 @@ func substitutionsString(m map[string]string, runRegion string) (string, bool) { subs = append(subs, fmt.Sprintf("%s=%s", k, v)) } - return strings.Join(subs, ","), false + return strings.Join(subs, ",") } diff --git a/internal/lifecycle/lifecycle.go b/internal/lifecycle/lifecycle.go index f3e5f20..213a056 100644 --- a/internal/lifecycle/lifecycle.go +++ b/internal/lifecycle/lifecycle.go @@ -23,8 +23,12 @@ import ( "os" "os/exec" "path/filepath" + "regexp" + "strings" ) +var gcrURLRegexp = regexp.MustCompile(`gcr.io/.+/\S+`) + // Lifecycle is a list of ordered exec.Cmd that should be run to execute a certain process. type Lifecycle []*exec.Cmd @@ -55,7 +59,7 @@ func NewLifecycle(sampleDir, serviceName, gcrURL, runRegion string, cloudBuildCo cloudBuildConfigE := err == nil if cloudBuildConfigE { - lifecycle, err := parseCloudBuildConfig(cloudBuildConfigPath, serviceName, gcrURL, runRegion, cloudBuildConfSubs) + lifecycle, err := getCloudBuildConfigLifecycle(cloudBuildConfigPath, serviceName, gcrURL, runRegion, cloudBuildConfSubs) if err == nil { log.Println("Using cloud build config file") return lifecycle, nil @@ -140,3 +144,38 @@ func buildDefaultJavaLifecycle(serviceName, gcrURL string) Lifecycle { return l } + +// replaceServiceName takes a terminal command string as input and replaces the Cloud Run service name, if any. +// If the user specified the service name in $CLOUD_RUN_SERVICE_NAME, it replaces that. Otherwise, as a failsafe, +// it detects whether the command is a gcloud run command and replaces the last argument that isn't a flag +// with the input service name. +func replaceServiceName(name string, args []string, serviceName string) error { + if !strings.Contains(name, "gcloud") { + return nil + } + + // Detects if the user specified the Cloud Run service name in an environment variable + for i := 0; i < len(args); i++ { + if args[i] == os.ExpandEnv("$CLOUD_RUN_SERVICE_NAME") { + args[i] = serviceName + return nil + } + } + + // Searches for specific gcloud keywords and takes service name from them + for i := 0; i < len(args)-1; i++ { + if args[i] == "deploy" || args[i] == "update" { + args[i+1] = serviceName + return nil + } + } + + // Provides a failsafe if neither of the above options work + for i := len(args) - 1; i >= 0; i-- { + if !strings.Contains(args[i], "--") { + args[i] = serviceName + break + } + } + return nil +} diff --git a/internal/lifecycle/readme.go b/internal/lifecycle/readme.go index 703420f..8c794b6 100644 --- a/internal/lifecycle/readme.go +++ b/internal/lifecycle/readme.go @@ -35,11 +35,6 @@ const ( ) var ( - gcloudCommandRegexp = regexp.MustCompile(`^gcloud\b`) - cloudRunCommandRegexp = regexp.MustCompile(`\brun\b`) - - gcrURLRegexp = regexp.MustCompile(`gcr.io/.+/\S+`) - mdCodeFenceStartRegexp = regexp.MustCompile("^\\w*`{3,}[^`]*$") errNoReadmeCodeBlocksFound = fmt.Errorf("lifecycle.extractCodeBlocks: no code blocks immediately preceded by %s found", codeTag) @@ -85,9 +80,14 @@ func (cb codeBlock) toCommands(serviceName, gcrURL string) ([]*exec.Cmd, error) line = os.ExpandEnv(line) line = gcrURLRegexp.ReplaceAllString(line, gcrURL) - line = replaceServiceName(line, serviceName) + sp := strings.Split(line, " ") + err := replaceServiceName(sp[0], sp[1:], serviceName) + if err != nil { + return nil, fmt.Errorf("[lifecycle.codeBlocksTolifecycle] replacing Cloud Run service name in README build and deploy commands: %w", err) + } + var cmd *exec.Cmd if sp[0] == "gcloud" { a := append(util.GcloudCommonFlags, sp[1:]...) @@ -205,40 +205,3 @@ func extractCodeBlocks(scanner *bufio.Scanner) ([]codeBlock, error) { return blocks, nil } - -// replaceServiceName takes a terminal command string as input and replaces the Cloud Run service name, if any. -// If the user specified the service name in $CLOUD_RUN_SERVICE_NAME, it replaces that. Otherwise, as a failsafe, -// it detects whether the command is a gcloud run command and replaces the last argument that isn't a flag -// with the input service name. -func replaceServiceName(command, serviceName string) string { - if !(gcloudCommandRegexp.MatchString(command) && cloudRunCommandRegexp.MatchString(command)) { - return command - } - - sp := strings.Split(command, " ") - - // Detects if the user specified the Cloud Run service name in an environment variable - for i := 0; i < len(sp); i++ { - if sp[i] == os.ExpandEnv("$CLOUD_RUN_SERVICE_NAME") { - sp[i] = serviceName - return strings.Join(sp, " ") - } - } - - // Searches for specific gcloud keywords and takes service name from them - for i := 0; i < len(sp)-1; i++ { - if sp[i] == "deploy" || sp[i] == "update" { - sp[i+1] = serviceName - return strings.Join(sp, " ") - } - } - - // Provides a failsafe if neither of the above options work - for i := len(sp) - 1; i >= 0; i-- { - if !strings.Contains(sp[i], "--") { - sp[i] = serviceName - break - } - } - return strings.Join(sp, " ") -} From 6b800ed099afa986de13070f6cfc613d106e8459 Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Mon, 17 Aug 2020 20:10:03 -0400 Subject: [PATCH 04/18] add comments, update README.md for Cloud Build config --- README.md | 14 ++++++++++++++ internal/lifecycle/cloud_build_config.go | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 91f46d0..dec390b 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,15 @@ Run Serverless Sample Tester by passing in the root directory of the sample you ./sst [target-dir] ``` +### Cloud Build Config +If you'd like, you can specify the build and deploy process for your sample in a [Cloud Build config file](https://cloud.google.com/cloud-build/docs/build-config) +with the name `cloudbuild.yaml` located in the root directory of your sample. Make sure to deploy to the Cloud Run fully +managed platform. + +It's required that you use the `_SST_RUN_REGION` [subustition](https://cloud.google.com/cloud-build/docs/configuring-builds/substitute-variable-values) +for the Cloud Run region you deploy your sample to. The substitution will be set as the same region specified by your +local gcloud installation's `run/region` gcloud property. + ### README parsing To parse build and deploy commands from your sample's README, include the following comment code tag before each gcloud command: @@ -79,3 +88,8 @@ what the name is, but it may not always be accurate. For example, if your README gcloud run deploy run-mysql --image gcr.io/[YOUR_PROJECT_ID]/run-mysql ``` then `$CLOUD_RUN_SERVICE_NAME` should be set to `run-mysql`. + +### Reasonable defaults +If a `cloudbuild.yaml` file isn't located in your sample's root directory and comment code tags aren't added to your +README, the program will fall back to reasonable defaults to build and deploy your sample to Cloud Run based on whether +your sample is java-based and doesn't have a Dockerfile or isn't. diff --git a/internal/lifecycle/cloud_build_config.go b/internal/lifecycle/cloud_build_config.go index 1522f23..0e3d2d8 100644 --- a/internal/lifecycle/cloud_build_config.go +++ b/internal/lifecycle/cloud_build_config.go @@ -23,7 +23,8 @@ import ( "strings" ) -// TODO: comment here +// runRegionSubstitution is the substitution used to specify which Cloud Run region Cloud Build configs will deploy +// samples to. const runRegionSubstitution = "_SST_RUN_REGION" // getCloudBuildConfigLifecycle returns a Lifecycle for the executing the provided Cloud Build config file. It creates From 4c5ed779a34f091e4f92b72a1a28d195d5707674 Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Mon, 17 Aug 2020 22:45:04 -0400 Subject: [PATCH 05/18] various cloud build config updates --- README.md | 13 ++++++++++++- internal/cmd/cmd.go | 10 ++++++---- internal/lifecycle/lifecycle.go | 7 +++++++ internal/util/tempfiles.go | 16 ++++++++++++++++ 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index dec390b..b7f5014 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,18 @@ If you'd like, you can specify the build and deploy process for your sample in a with the name `cloudbuild.yaml` located in the root directory of your sample. Make sure to deploy to the Cloud Run fully managed platform. -It's required that you use the `_SST_RUN_REGION` [subustition](https://cloud.google.com/cloud-build/docs/configuring-builds/substitute-variable-values) +You can specify the substitutions you'd like passed to your Cloud Build config by either setting the +`SST_CLOUD_BUILD_SUBS` environment variable in JSON format or the `--cloud-build-subs` flag in comma-separated format +when calling the `sst` executable. For example: + +```bash +export SST_CLOUD_BUILD_SUBS="{\"_FOO\": \"hello\",\"_BAR\": \"world\"}" +sst [sample-dir] # or + +sst [sample-dir] --cloud-build-subs="_FOO=hello,_BAR=world" +``` + +It's required that you use the `_SST_RUN_REGION` [substitution](https://cloud.google.com/cloud-build/docs/configuring-builds/substitute-variable-values) for the Cloud Run region you deploy your sample to. The substitution will be set as the same region specified by your local gcloud installation's `run/region` gcloud property. diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index a40c005..1382480 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -25,12 +25,14 @@ import ( "path/filepath" ) +// InitConfig initializes binds command line flags and environment variables to viper configuration keys. This should +// be called before the provided cobra.Command is executed. func InitConfig(rootCmd *cobra.Command) { viper.SetEnvPrefix("sst") - viper.BindEnv("substitutions") - rootCmd.PersistentFlags().StringToString("substitutions", map[string]string{}, "dsfasd") - viper.BindPFlag("substitutions", rootCmd.PersistentFlags().Lookup("substitutions")) + viper.BindEnv("cloud_build_subs") + rootCmd.PersistentFlags().StringToString("cloud-build-subs", map[string]string{}, "") + viper.BindPFlag("cloud_build_subs", rootCmd.PersistentFlags().Lookup("cloud-build-subs")) } // Root is responsible for the root command. It handles the application flow. @@ -42,7 +44,7 @@ func Root(cmd *cobra.Command, args []string) error { } log.Println("Setting up configuration values") - s, err := sample.NewSample(sampleDir, viper.GetStringMapString("substitutions")) + s, err := sample.NewSample(sampleDir, viper.GetStringMapString("cloud_build_subs")) if err != nil { return err } diff --git a/internal/lifecycle/lifecycle.go b/internal/lifecycle/lifecycle.go index 213a056..4a49147 100644 --- a/internal/lifecycle/lifecycle.go +++ b/internal/lifecycle/lifecycle.go @@ -172,7 +172,14 @@ func replaceServiceName(name string, args []string, serviceName string) error { // Provides a failsafe if neither of the above options work for i := len(args) - 1; i >= 0; i-- { +<<<<<<< HEAD if !strings.Contains(args[i], "--") { +======= + // Check if arg before is a flag without an argument + lastArgFlag := i-1 != -1 && (strings.HasPrefix(args[i], "--") && !strings.Contains(args[i], "=")) + + if args[i] == s && !lastArgFlag { +>>>>>>> various cloud build config updates args[i] = serviceName break } diff --git a/internal/util/tempfiles.go b/internal/util/tempfiles.go index 0691ea7..7883b4e 100644 --- a/internal/util/tempfiles.go +++ b/internal/util/tempfiles.go @@ -1,3 +1,17 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package util import ( @@ -7,6 +21,8 @@ import ( "os" ) +// tempFiles is a slice of pointers to temporary files created throughout the course of the program. Users should call +// RemoveTempFiles at the end of the program. var tempFiles []*os.File func CreateTempFile() (*os.File, error) { From ded600ce569898ef8dfba5812f2e7e9316f2b17f Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Mon, 17 Aug 2020 22:56:37 -0400 Subject: [PATCH 06/18] minor README.md update --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b7f5014..4d2131d 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,8 @@ with the name `cloudbuild.yaml` located in the root directory of your sample. Ma managed platform. You can specify the substitutions you'd like passed to your Cloud Build config by either setting the -`SST_CLOUD_BUILD_SUBS` environment variable in JSON format or the `--cloud-build-subs` flag in comma-separated format -when calling the `sst` executable. For example: +`SST_CLOUD_BUILD_SUBS` environment variable in a JSON format or the `--cloud-build-subs` flag in a comma-separated +format when calling the `sst` executable. For example: ```bash export SST_CLOUD_BUILD_SUBS="{\"_FOO\": \"hello\",\"_BAR\": \"world\"}" @@ -52,7 +52,7 @@ sst [sample-dir] --cloud-build-subs="_FOO=hello,_BAR=world" It's required that you use the `_SST_RUN_REGION` [substitution](https://cloud.google.com/cloud-build/docs/configuring-builds/substitute-variable-values) for the Cloud Run region you deploy your sample to. The substitution will be set as the same region specified by your -local gcloud installation's `run/region` gcloud property. +local gcloud installation's `run/region` property. ### README parsing To parse build and deploy commands from your sample's README, include the following comment code tag before each gcloud command: From de1a9c85a7f338a5129667757c893466cb26e4f2 Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Tue, 18 Aug 2020 19:12:10 -0400 Subject: [PATCH 07/18] remove tempfiles.go in favor of cleanup function returns --- internal/cmd/cmd.go | 5 ++- internal/lifecycle/cloud_build_config.go | 31 ++++++++++------ internal/lifecycle/lifecycle.go | 27 ++++++-------- internal/sample/sample.go | 20 +++++----- internal/util/tempfiles.go | 47 ------------------------ 5 files changed, 46 insertions(+), 84 deletions(-) delete mode 100644 internal/util/tempfiles.go diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 1382480..69ec264 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -44,10 +44,13 @@ func Root(cmd *cobra.Command, args []string) error { } log.Println("Setting up configuration values") - s, err := sample.NewSample(sampleDir, viper.GetStringMapString("cloud_build_subs")) + s, cleanup, err := sample.NewSample(sampleDir, viper.GetStringMapString("cloud_build_subs")) if err != nil { return err } + if cleanup != nil { + defer cleanup() + } log.Println("Loading test endpoints") swagger := util.LoadTestEndpoints() diff --git a/internal/lifecycle/cloud_build_config.go b/internal/lifecycle/cloud_build_config.go index 0e3d2d8..d956637 100644 --- a/internal/lifecycle/cloud_build_config.go +++ b/internal/lifecycle/cloud_build_config.go @@ -19,6 +19,8 @@ import ( "github.com/GoogleCloudPlatform/serverless-sample-tester/internal/util" "gopkg.in/yaml.v2" "io/ioutil" + "log" + "os" "os/exec" "strings" ) @@ -30,18 +32,19 @@ const runRegionSubstitution = "_SST_RUN_REGION" // getCloudBuildConfigLifecycle returns a Lifecycle for the executing the provided Cloud Build config file. It creates // and uses a temporary copy of the file where it replaces the Cloud Run service names and Container Registry tags with // the provided inputs. It provides also passes in the provided substitutions as well a runRegionSubstitution with the -// provided region. -func getCloudBuildConfigLifecycle(filename, serviceName, gcrURL, runRegion string, substitutions map[string]string) (Lifecycle, error) { +// provided region. Also returns a function that removes the temp file created while making Lifecycle. This function +// should be called after Lifecycle is done executing. +func getCloudBuildConfigLifecycle(filename, serviceName, gcrURL, runRegion string, substitutions map[string]string) (Lifecycle, func(), error) { config := make(map[string]interface{}) buildConfigBytes, err := ioutil.ReadFile(filename) if err != nil { - return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] reading Cloud Build config file: %w", err) + return nil, nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] reading Cloud Build config file: %w", err) } err = yaml.Unmarshal(buildConfigBytes, &config) if err != nil { - return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] unmarshaling Cloud Build config file: %w", err) + return nil, nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] unmarshaling Cloud Build config file: %w", err) } // Replace Cloud Run service names and Container Registry URLs @@ -57,7 +60,7 @@ func getCloudBuildConfigLifecycle(filename, serviceName, gcrURL, runRegion strin prog := config["steps"].([]interface{})[stepIndex].(map[interface{}]interface{})["name"].(string) err := replaceServiceName(prog, args, serviceName) if err != nil { - return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] replacing Cloud Run service name in Cloud Build config step args: %w", err) + return nil, nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] replacing Cloud Run service name in Cloud Build config step args: %w", err) } config["steps"].([]interface{})[stepIndex].(map[interface{}]interface{})["args"] = args @@ -65,22 +68,28 @@ func getCloudBuildConfigLifecycle(filename, serviceName, gcrURL, runRegion strin configMarshalBytes, err := yaml.Marshal(&config) if err != nil { - return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] marshaling modified Cloud Build config: %w", err) + return nil, nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] marshaling modified Cloud Build config: %w", err) } - tempBuildConfigFile, err := util.CreateTempFile() + tempBuildConfigFile, err := ioutil.TempFile("", "example") if err != nil { - return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] creating temporary file: %w", err) + return nil, nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] creating Temp File: %w\n", err) + } + cleanup := func() { + err := os.Remove(tempBuildConfigFile.Name()) + if err != nil { + log.Printf("Error removing Temp File for Cloud Build config: %v\n", err) + } } if _, err := tempBuildConfigFile.Write(configMarshalBytes); err != nil { - return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] writing to temporary file: %w", err) + return nil, cleanup, fmt.Errorf("[lifecycle.parseCloudBuildConfig] writing to temporary file: %w", err) } if err := tempBuildConfigFile.Close(); err != nil { - return nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] closing temporary file: %w", err) + return nil, cleanup, fmt.Errorf("[lifecycle.parseCloudBuildConfig] closing temporary file: %w", err) } - return buildCloudBuildConfigLifecycle(tempBuildConfigFile.Name(), runRegion, substitutions), nil + return buildCloudBuildConfigLifecycle(tempBuildConfigFile.Name(), runRegion, substitutions), cleanup, nil } // buildCloudBuildConfigLifecycle returns a Lifecycle with a single command that calls gcloud builds subit and passes diff --git a/internal/lifecycle/lifecycle.go b/internal/lifecycle/lifecycle.go index 4a49147..dc5e701 100644 --- a/internal/lifecycle/lifecycle.go +++ b/internal/lifecycle/lifecycle.go @@ -50,8 +50,10 @@ func (l Lifecycle) Execute(commandsDir string) error { // NewLifecycle tries to parse the different options provided for build and deploy command configuration. If none of // those options are set up, it falls back to reasonable defaults based on whether the sample is java-based -// (has a pom.xml) that doesn't have a Dockerfile or isn't. -func NewLifecycle(sampleDir, serviceName, gcrURL, runRegion string, cloudBuildConfSubs map[string]string) (Lifecycle, error) { +// (has a pom.xml) that doesn't have a Dockerfile or isn't. Also returns a function that cleans up any created local +// resources (e.g. temp files) created while making creating this Lifecycle. This function should be called after this +// Lifecycle is done executing. +func NewLifecycle(sampleDir, serviceName, gcrURL, runRegion string, cloudBuildConfSubs map[string]string) (Lifecycle, func(), error) { // First try Cloud Build Config file cloudBuildConfigPath := fmt.Sprintf("%s/cloudbuild.yaml", sampleDir) @@ -59,13 +61,13 @@ func NewLifecycle(sampleDir, serviceName, gcrURL, runRegion string, cloudBuildCo cloudBuildConfigE := err == nil if cloudBuildConfigE { - lifecycle, err := getCloudBuildConfigLifecycle(cloudBuildConfigPath, serviceName, gcrURL, runRegion, cloudBuildConfSubs) + lifecycle, cleanup, err := getCloudBuildConfigLifecycle(cloudBuildConfigPath, serviceName, gcrURL, runRegion, cloudBuildConfSubs) if err == nil { log.Println("Using cloud build config file") - return lifecycle, nil + return lifecycle, cleanup, nil } - return nil, fmt.Errorf("[lifecycle.NewLifecycle] using cloud build config file %s: %v\n", cloudBuildConfigPath, err) + return nil, cleanup, fmt.Errorf("[lifecycle.NewLifecycle] using cloud build config file %s: %v\n", cloudBuildConfigPath, err) } // Then try README parsing @@ -85,11 +87,11 @@ func NewLifecycle(sampleDir, serviceName, gcrURL, runRegion string, cloudBuildCo log.Println("README.md location: " + readmePath) if err == nil { log.Println("Using build and deploy commands found in README.md") - return lifecycle, nil + return lifecycle, nil, nil } if !errors.Is(err, errNoReadmeCodeBlocksFound) { - return nil, fmt.Errorf("lifecycle.parseREADME: %s: %w", readmePath, err) + return nil, nil, fmt.Errorf("lifecycle.parseREADME: %s: %w", readmePath, err) } log.Printf("No code blocks immediately preceded by %s found in README.md\n", codeTag) @@ -109,11 +111,11 @@ func NewLifecycle(sampleDir, serviceName, gcrURL, runRegion string, cloudBuildCo if pomE && !dockerfileE { log.Println("Using default build and deploy commands for java samples without a Dockerfile") - return buildDefaultJavaLifecycle(serviceName, gcrURL), nil + return buildDefaultJavaLifecycle(serviceName, gcrURL), nil, nil } log.Println("Using default build and deploy commands for non-java samples or java samples with a Dockerfile") - return buildDefaultLifecycle(serviceName, gcrURL), nil + return buildDefaultLifecycle(serviceName, gcrURL), nil, nil } // buildDefaultLifecycle builds a build and deploy command lifecycle with reasonable defaults for a non-Java @@ -172,14 +174,7 @@ func replaceServiceName(name string, args []string, serviceName string) error { // Provides a failsafe if neither of the above options work for i := len(args) - 1; i >= 0; i-- { -<<<<<<< HEAD if !strings.Contains(args[i], "--") { -======= - // Check if arg before is a flag without an argument - lastArgFlag := i-1 != -1 && (strings.HasPrefix(args[i], "--") && !strings.Contains(args[i], "=")) - - if args[i] == s && !lastArgFlag { ->>>>>>> various cloud build config updates args[i] = serviceName break } diff --git a/internal/sample/sample.go b/internal/sample/sample.go index a56de6d..e54628e 100644 --- a/internal/sample/sample.go +++ b/internal/sample/sample.go @@ -46,20 +46,22 @@ type Sample struct { runRegion string } -// NewSample creates a new sample object for the sample located in the provided local directory. -func NewSample(dir string, cloudBuildConfSubs map[string]string) (*Sample, error) { +// NewSample creates a new sample object for the sample located in the provided local directory. Also returns a function +// that cleans up any created local resources (e.g. temp files) created while making creating this object. This function +// should be called after this sample object is no longer needed. +func NewSample(dir string, cloudBuildConfSubs map[string]string) (*Sample, func(), error) { name := sampleName(dir) containerTag, err := cloudContainerImageTag(name, dir) if err != nil { - return nil, fmt.Errorf("sample.cloudContainerImageTag: %s %s: %w", name, dir, err) + return nil, nil, fmt.Errorf("sample.cloudContainerImageTag: %s %s: %w", name, dir, err) } a := append(util.GcloudCommonFlags, "config", "get-value", "core/project") projectID, err := util.ExecCommand(exec.Command("gcloud", a...), dir) if err != nil { - return nil, fmt.Errorf("getting gcloud default project: %w", err) + return nil, nil, fmt.Errorf("getting gcloud default project: %w", err) } cloudContainerImageURL := fmt.Sprintf("gcr.io/%s/%s", projectID, containerTag) @@ -67,18 +69,18 @@ func NewSample(dir string, cloudBuildConfSubs map[string]string) (*Sample, error runRegion, err := util.ExecCommand(exec.Command("gcloud", a...), dir) if err != nil { - return nil, fmt.Errorf("[sample.NewSample] getting gcloud cloud run default region: %w", err) + return nil, nil, fmt.Errorf("getting gcloud cloud run default region: %w", err) } serviceName, err := gcloud.ServiceName(name) if err != nil { - return nil, fmt.Errorf("gcloud.ServiceName: %s sample: %w", name, err) + return nil, nil, fmt.Errorf("gcloud.ServiceName: %s sample: %w", name, err) } service := gcloud.CloudRunService{Name: serviceName} - buildDeployLifecycle, err := lifecycle.NewLifecycle(dir, service.Name, cloudContainerImageURL, runRegion, cloudBuildConfSubs) + buildDeployLifecycle, cleanup, err := lifecycle.NewLifecycle(dir, service.Name, cloudContainerImageURL, runRegion, cloudBuildConfSubs) if err != nil { - return nil, fmt.Errorf("lifecycle.NewLifecycle: %w", err) + return nil, cleanup, fmt.Errorf("lifecycle.NewLifecycle: %w", err) } s := &Sample{ @@ -89,7 +91,7 @@ func NewSample(dir string, cloudBuildConfSubs map[string]string) (*Sample, error cloudContainerImageURL: cloudContainerImageURL, runRegion: runRegion, } - return s, nil + return s, cleanup, nil } // sampleName computes a sample name for a sample object. Right now, it's defined as a shortened version of the sample's diff --git a/internal/util/tempfiles.go b/internal/util/tempfiles.go deleted file mode 100644 index 7883b4e..0000000 --- a/internal/util/tempfiles.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package util - -import ( - "fmt" - "io/ioutil" - "log" - "os" -) - -// tempFiles is a slice of pointers to temporary files created throughout the course of the program. Users should call -// RemoveTempFiles at the end of the program. -var tempFiles []*os.File - -func CreateTempFile() (*os.File, error) { - tempFile, err := ioutil.TempFile("", "example") - if err != nil { - return tempFile, fmt.Errorf("[util.CreateTempFile] creating temp file: %w\n", err) - } - - tempFiles = append(tempFiles, tempFile) - return tempFile, nil -} - -func RemoveTempFiles() error { - for _, tempFile := range tempFiles { - err := os.Remove(tempFile.Name()) - if err != nil { - log.Printf("Error removing Temp File: %v\n", err) - } - } - - return nil -} From c4a998accf0aadeebf3d29a4c05f3bbc62fb0b9a Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Wed, 19 Aug 2020 18:01:54 -0400 Subject: [PATCH 08/18] refactor temp file cleanup logic --- internal/lifecycle/cloud_build_config.go | 6 ++++-- internal/lifecycle/lifecycle.go | 2 +- internal/sample/sample.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/lifecycle/cloud_build_config.go b/internal/lifecycle/cloud_build_config.go index d956637..89cc82c 100644 --- a/internal/lifecycle/cloud_build_config.go +++ b/internal/lifecycle/cloud_build_config.go @@ -83,10 +83,12 @@ func getCloudBuildConfigLifecycle(filename, serviceName, gcrURL, runRegion strin } if _, err := tempBuildConfigFile.Write(configMarshalBytes); err != nil { - return nil, cleanup, fmt.Errorf("[lifecycle.parseCloudBuildConfig] writing to temporary file: %w", err) + cleanup() + return nil, nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] writing to temporary file: %w", err) } if err := tempBuildConfigFile.Close(); err != nil { - return nil, cleanup, fmt.Errorf("[lifecycle.parseCloudBuildConfig] closing temporary file: %w", err) + cleanup() + return nil, nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] closing temporary file: %w", err) } return buildCloudBuildConfigLifecycle(tempBuildConfigFile.Name(), runRegion, substitutions), cleanup, nil diff --git a/internal/lifecycle/lifecycle.go b/internal/lifecycle/lifecycle.go index dc5e701..853853e 100644 --- a/internal/lifecycle/lifecycle.go +++ b/internal/lifecycle/lifecycle.go @@ -67,7 +67,7 @@ func NewLifecycle(sampleDir, serviceName, gcrURL, runRegion string, cloudBuildCo return lifecycle, cleanup, nil } - return nil, cleanup, fmt.Errorf("[lifecycle.NewLifecycle] using cloud build config file %s: %v\n", cloudBuildConfigPath, err) + return nil, nil, fmt.Errorf("[lifecycle.NewLifecycle] using cloud build config file %s: %v\n", cloudBuildConfigPath, err) } // Then try README parsing diff --git a/internal/sample/sample.go b/internal/sample/sample.go index e54628e..c2783be 100644 --- a/internal/sample/sample.go +++ b/internal/sample/sample.go @@ -80,7 +80,7 @@ func NewSample(dir string, cloudBuildConfSubs map[string]string) (*Sample, func( buildDeployLifecycle, cleanup, err := lifecycle.NewLifecycle(dir, service.Name, cloudContainerImageURL, runRegion, cloudBuildConfSubs) if err != nil { - return nil, cleanup, fmt.Errorf("lifecycle.NewLifecycle: %w", err) + return nil, nil, fmt.Errorf("lifecycle.NewLifecycle: %w", err) } s := &Sample{ From c3fafc37d97d289954230e1e7e5dbcd24a3b9bc7 Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Wed, 19 Aug 2020 18:11:19 -0400 Subject: [PATCH 09/18] minor code refactor in internal/lifecycle.go --- internal/lifecycle/lifecycle.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/lifecycle/lifecycle.go b/internal/lifecycle/lifecycle.go index 853853e..f410c93 100644 --- a/internal/lifecycle/lifecycle.go +++ b/internal/lifecycle/lifecycle.go @@ -57,10 +57,7 @@ func NewLifecycle(sampleDir, serviceName, gcrURL, runRegion string, cloudBuildCo // First try Cloud Build Config file cloudBuildConfigPath := fmt.Sprintf("%s/cloudbuild.yaml", sampleDir) - _, err := os.Stat(cloudBuildConfigPath) - cloudBuildConfigE := err == nil - - if cloudBuildConfigE { + if _, err := os.Stat(cloudBuildConfigPath); err == nil { lifecycle, cleanup, err := getCloudBuildConfigLifecycle(cloudBuildConfigPath, serviceName, gcrURL, runRegion, cloudBuildConfSubs) if err == nil { log.Println("Using cloud build config file") @@ -103,7 +100,7 @@ func NewLifecycle(sampleDir, serviceName, gcrURL, runRegion string, cloudBuildCo pomPath := filepath.Join(sampleDir, "pom.xml") dockerfilePath := filepath.Join(sampleDir, "Dockerfile") - _, err = os.Stat(pomPath) + _, err := os.Stat(pomPath) pomE := err == nil _, err = os.Stat(dockerfilePath) From 9c101844d34229bc68b9e6e3de58743c16bba0b9 Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Wed, 19 Aug 2020 18:32:12 -0400 Subject: [PATCH 10/18] README.md format update --- README.md | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4d2131d..b68e227 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,16 @@ Run Serverless Sample Tester by passing in the root directory of the sample you ./sst [target-dir] ``` + +## Build and Deploying Your Sample +This program allows you to specify how you would like it to build and deploy your sample to Cloud Run, fully managed, +in three ways. From highest to lowest, this program uses the following precedence order to find your build and deploy +instructions: + +1. Cloud Build Config file +1. README parsing +1. Defaults + ### Cloud Build Config If you'd like, you can specify the build and deploy process for your sample in a [Cloud Build config file](https://cloud.google.com/cloud-build/docs/build-config) with the name `cloudbuild.yaml` located in the root directory of your sample. Make sure to deploy to the Cloud Run fully @@ -100,7 +110,11 @@ gcloud run deploy run-mysql --image gcr.io/[YOUR_PROJECT_ID]/run-mysql ``` then `$CLOUD_RUN_SERVICE_NAME` should be set to `run-mysql`. -### Reasonable defaults -If a `cloudbuild.yaml` file isn't located in your sample's root directory and comment code tags aren't added to your -README, the program will fall back to reasonable defaults to build and deploy your sample to Cloud Run based on whether -your sample is java-based and doesn't have a Dockerfile or isn't. +### Defaults +If your sample is Java-based and doesn't have a Dockerfile, your sample will be built and pushed the Container Registry +using `mvn compile com.google.cloud.tools:jib-maven-plugin:2.0.0:build -Dimage=[image_tag]`. + +Otherwise, your sample will be build and pushed to the Container Registry using `gcloud builds submit --tag=[image_tag]`. + +In both cases, `gcloud run deploy --image=[image_tag] --platform=managed`, will be used to deploy the container image to +Cloud Run. From 5bbe9ea21a908ddb3dfad0f0cc4d0da2e5363b6a Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Wed, 19 Aug 2020 18:32:44 -0400 Subject: [PATCH 11/18] explicitly set viper default for cloud build substitutions --- internal/cmd/cmd.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 69ec264..14177f6 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -30,6 +30,7 @@ import ( func InitConfig(rootCmd *cobra.Command) { viper.SetEnvPrefix("sst") + viper.SetDefault("cloud_build_subs", map[string]string{}) viper.BindEnv("cloud_build_subs") rootCmd.PersistentFlags().StringToString("cloud-build-subs", map[string]string{}, "") viper.BindPFlag("cloud_build_subs", rootCmd.PersistentFlags().Lookup("cloud-build-subs")) From 63f0e32bba22589fc29e709479c302325ae70f6b Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Wed, 19 Aug 2020 19:01:38 -0400 Subject: [PATCH 12/18] update error message format and temp file format --- internal/lifecycle/cloud_build_config.go | 16 ++++++++-------- internal/lifecycle/lifecycle.go | 2 +- internal/lifecycle/readme.go | 2 +- internal/sample/sample.go | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/internal/lifecycle/cloud_build_config.go b/internal/lifecycle/cloud_build_config.go index 89cc82c..479ac89 100644 --- a/internal/lifecycle/cloud_build_config.go +++ b/internal/lifecycle/cloud_build_config.go @@ -39,12 +39,12 @@ func getCloudBuildConfigLifecycle(filename, serviceName, gcrURL, runRegion strin buildConfigBytes, err := ioutil.ReadFile(filename) if err != nil { - return nil, nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] reading Cloud Build config file: %w", err) + return nil, nil, fmt.Errorf("ioutil.ReadFile: %w", err) } err = yaml.Unmarshal(buildConfigBytes, &config) if err != nil { - return nil, nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] unmarshaling Cloud Build config file: %w", err) + return nil, nil, fmt.Errorf("yaml.Unmarshal: %s: %w", filename, err) } // Replace Cloud Run service names and Container Registry URLs @@ -60,7 +60,7 @@ func getCloudBuildConfigLifecycle(filename, serviceName, gcrURL, runRegion strin prog := config["steps"].([]interface{})[stepIndex].(map[interface{}]interface{})["name"].(string) err := replaceServiceName(prog, args, serviceName) if err != nil { - return nil, nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] replacing Cloud Run service name in Cloud Build config step args: %w", err) + return nil, nil, fmt.Errorf("replacing Cloud Run service name in %s step args: %w", filename, err) } config["steps"].([]interface{})[stepIndex].(map[interface{}]interface{})["args"] = args @@ -68,12 +68,12 @@ func getCloudBuildConfigLifecycle(filename, serviceName, gcrURL, runRegion strin configMarshalBytes, err := yaml.Marshal(&config) if err != nil { - return nil, nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] marshaling modified Cloud Build config: %w", err) + return nil, nil, fmt.Errorf("yaml.Marshal: modified %s: %w", filename, err) } - tempBuildConfigFile, err := ioutil.TempFile("", "example") + tempBuildConfigFile, err := ioutil.TempFile("", "cloudbuild.*.yaml") if err != nil { - return nil, nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] creating Temp File: %w\n", err) + return nil, nil, fmt.Errorf("ioutil.TempFile: %w\n", err) } cleanup := func() { err := os.Remove(tempBuildConfigFile.Name()) @@ -84,11 +84,11 @@ func getCloudBuildConfigLifecycle(filename, serviceName, gcrURL, runRegion strin if _, err := tempBuildConfigFile.Write(configMarshalBytes); err != nil { cleanup() - return nil, nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] writing to temporary file: %w", err) + return nil, nil, fmt.Errorf("os.File.Write: TempFile %s: %w", tempBuildConfigFile.Name(), err) } if err := tempBuildConfigFile.Close(); err != nil { cleanup() - return nil, nil, fmt.Errorf("[lifecycle.parseCloudBuildConfig] closing temporary file: %w", err) + return nil, nil, fmt.Errorf("os.File.Close: TempFile %s: %w", tempBuildConfigFile.Name(), err) } return buildCloudBuildConfigLifecycle(tempBuildConfigFile.Name(), runRegion, substitutions), cleanup, nil diff --git a/internal/lifecycle/lifecycle.go b/internal/lifecycle/lifecycle.go index f410c93..0929535 100644 --- a/internal/lifecycle/lifecycle.go +++ b/internal/lifecycle/lifecycle.go @@ -64,7 +64,7 @@ func NewLifecycle(sampleDir, serviceName, gcrURL, runRegion string, cloudBuildCo return lifecycle, cleanup, nil } - return nil, nil, fmt.Errorf("[lifecycle.NewLifecycle] using cloud build config file %s: %v\n", cloudBuildConfigPath, err) + return nil, nil, fmt.Errorf("lifecycle.getCloudBuildConfigLifecycle: %s: %w\n", cloudBuildConfigPath, err) } // Then try README parsing diff --git a/internal/lifecycle/readme.go b/internal/lifecycle/readme.go index 8c794b6..a7f2cd1 100644 --- a/internal/lifecycle/readme.go +++ b/internal/lifecycle/readme.go @@ -85,7 +85,7 @@ func (cb codeBlock) toCommands(serviceName, gcrURL string) ([]*exec.Cmd, error) err := replaceServiceName(sp[0], sp[1:], serviceName) if err != nil { - return nil, fmt.Errorf("[lifecycle.codeBlocksTolifecycle] replacing Cloud Run service name in README build and deploy commands: %w", err) + return nil, fmt.Errorf("lifecycle.replaceServiceName: %s: %w", line, err) } var cmd *exec.Cmd diff --git a/internal/sample/sample.go b/internal/sample/sample.go index c2783be..aac9292 100644 --- a/internal/sample/sample.go +++ b/internal/sample/sample.go @@ -54,7 +54,7 @@ func NewSample(dir string, cloudBuildConfSubs map[string]string) (*Sample, func( containerTag, err := cloudContainerImageTag(name, dir) if err != nil { - return nil, nil, fmt.Errorf("sample.cloudContainerImageTag: %s %s: %w", name, dir, err) + return nil, nil, fmt.Errorf("sample.cloudContainerImageTag: sample name: %s, sample dir: %s: %w", name, dir, err) } a := append(util.GcloudCommonFlags, "config", "get-value", "core/project") @@ -74,7 +74,7 @@ func NewSample(dir string, cloudBuildConfSubs map[string]string) (*Sample, func( serviceName, err := gcloud.ServiceName(name) if err != nil { - return nil, nil, fmt.Errorf("gcloud.ServiceName: %s sample: %w", name, err) + return nil, nil, fmt.Errorf("gcloud.ServiceName: sample name: %s: %w", name, err) } service := gcloud.CloudRunService{Name: serviceName} From 4f275e40162fd6cd4da5c5325b907260fc94d677 Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Thu, 20 Aug 2020 15:44:53 -0400 Subject: [PATCH 13/18] clarify defaults section README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b68e227..d92c3c9 100644 --- a/README.md +++ b/README.md @@ -111,10 +111,10 @@ gcloud run deploy run-mysql --image gcr.io/[YOUR_PROJECT_ID]/run-mysql then `$CLOUD_RUN_SERVICE_NAME` should be set to `run-mysql`. ### Defaults -If your sample is Java-based and doesn't have a Dockerfile, your sample will be built and pushed the Container Registry -using `mvn compile com.google.cloud.tools:jib-maven-plugin:2.0.0:build -Dimage=[image_tag]`. +If your sample is Java-based (has a `pom.xml` file in its root directory) and has a `Dockerfile` in its root directory, +your sample will be built and pushed the Container Registry using `mvn compile com.google.cloud.tools:jib-maven-plugin:2.0.0:build -Dimage=[image_tag]`. -Otherwise, your sample will be build and pushed to the Container Registry using `gcloud builds submit --tag=[image_tag]`. +Otherwise, your sample will be built and pushed to the Container Registry using `gcloud builds submit --tag=[image_tag]`. In both cases, `gcloud run deploy --image=[image_tag] --platform=managed`, will be used to deploy the container image to Cloud Run. From 7e45dc9683b28cee84a680a90735714288dfe311 Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Sat, 22 Aug 2020 16:02:32 -0400 Subject: [PATCH 14/18] update to replaceServiceName algorithm --- internal/lifecycle/lifecycle.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/internal/lifecycle/lifecycle.go b/internal/lifecycle/lifecycle.go index 0929535..ffa7123 100644 --- a/internal/lifecycle/lifecycle.go +++ b/internal/lifecycle/lifecycle.go @@ -153,12 +153,23 @@ func replaceServiceName(name string, args []string, serviceName string) error { return nil } + var runCmd bool + // Detects if the user specified the Cloud Run service name in an environment variable for i := 0; i < len(args); i++ { if args[i] == os.ExpandEnv("$CLOUD_RUN_SERVICE_NAME") { args[i] = serviceName return nil } + + if args[i] == "run" { + runCmd = true + break + } + } + + if !runCmd { + return nil } // Searches for specific gcloud keywords and takes service name from them From 686765c33b2901510b5f8ba2c9c6b9e8ffb38eb6 Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Sat, 22 Aug 2020 16:04:35 -0400 Subject: [PATCH 15/18] go mod tidy --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index 9008205..b264c3e 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,5 @@ require ( github.com/getkin/kin-openapi v0.18.0 github.com/spf13/cobra v1.0.0 github.com/spf13/viper v1.7.1 - github.com/spf13/pflag v1.0.3 gopkg.in/yaml.v2 v2.3.0 ) From 77d2e3dbdaeeffe08fd1084a5dfdf6cae9b44984 Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Sat, 22 Aug 2020 16:09:02 -0400 Subject: [PATCH 16/18] fix rebase --- cmd/root.go | 13 ++++--- internal/cmd/cmd.go | 94 --------------------------------------------- 2 files changed, 8 insertions(+), 99 deletions(-) delete mode 100644 internal/cmd/cmd.go diff --git a/cmd/root.go b/cmd/root.go index 94ae4fb..cccdd93 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -27,11 +27,11 @@ import ( var ( rootCmd = &cobra.Command{ - Use: "sst [sample-dir]", - Short: "An end-to-end tester for GCP samples", - Args: cobra.ExactArgs(1), + Use: "sst [sample-dir]", + Short: "An end-to-end tester for GCP samples", + Args: cobra.ExactArgs(1), SilenceErrors: true, - SilenceUsage: true, + SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { // Parse sample directory from command line argument sampleDir, err := filepath.Abs(filepath.Dir(args[0])) @@ -40,10 +40,13 @@ var ( } log.Println("Setting up configuration values") - s, err := sample.NewSample(sampleDir, viper.GetStringMapString("cloud_build_subs")) + s, cleanup, err := sample.NewSample(sampleDir, viper.GetStringMapString("cloud_build_subs")) if err != nil { return err } + if cleanup != nil { + defer cleanup() + } log.Println("Loading test endpoints") swagger := util.LoadTestEndpoints() diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go deleted file mode 100644 index 14177f6..0000000 --- a/internal/cmd/cmd.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "fmt" - "github.com/GoogleCloudPlatform/serverless-sample-tester/internal/sample" - "github.com/GoogleCloudPlatform/serverless-sample-tester/internal/util" - "github.com/spf13/cobra" - "github.com/spf13/viper" - "log" - "os/exec" - "path/filepath" -) - -// InitConfig initializes binds command line flags and environment variables to viper configuration keys. This should -// be called before the provided cobra.Command is executed. -func InitConfig(rootCmd *cobra.Command) { - viper.SetEnvPrefix("sst") - - viper.SetDefault("cloud_build_subs", map[string]string{}) - viper.BindEnv("cloud_build_subs") - rootCmd.PersistentFlags().StringToString("cloud-build-subs", map[string]string{}, "") - viper.BindPFlag("cloud_build_subs", rootCmd.PersistentFlags().Lookup("cloud-build-subs")) -} - -// Root is responsible for the root command. It handles the application flow. -func Root(cmd *cobra.Command, args []string) error { - // Parse sample directory from command line argument - sampleDir, err := filepath.Abs(filepath.Dir(args[0])) - if err != nil { - return err - } - - log.Println("Setting up configuration values") - s, cleanup, err := sample.NewSample(sampleDir, viper.GetStringMapString("cloud_build_subs")) - if err != nil { - return err - } - if cleanup != nil { - defer cleanup() - } - - log.Println("Loading test endpoints") - swagger := util.LoadTestEndpoints() - - log.Println("Building and deploying sample to Cloud Run") - err = s.BuildDeployLifecycle.Execute(s.Dir) - defer s.Service.Delete(s.Dir) - defer s.DeleteCloudContainerImage() - if err != nil { - return fmt.Errorf("[cmd.Root] building and deploying sample to Cloud Run: %w", err) - } - - log.Println("Getting identity token for gcloud auhtorized account") - var identToken string - - a := append(util.GcloudCommonFlags, "auth", "print-identity-token") - identToken, err = util.ExecCommand(exec.Command("gcloud", a...), s.Dir) - - if err != nil { - return fmt.Errorf("[cmd.Root] getting identity token for gcloud auhtorized account: %w", err) - } - - log.Println("Checking endpoints for expected results") - serviceURL, err := s.Service.URL(s.Dir) - if err != nil { - return fmt.Errorf("[cmd.Root] getting Cloud Run service URL: %w", err) - } - - log.Println("Validating Cloud Run service endpoints for expected status codes") - allTestsPassed, err := util.ValidateEndpoints(serviceURL, &swagger.Paths, identToken) - if err != nil { - return fmt.Errorf("[cmd.Root] validating Cloud Run service endpoints for expected status codes: %w", err) - } - - if !allTestsPassed { - return fmt.Errorf("all tests did not pass") - } - - return nil -} From a13dafae205199ec87cd28b10bac15a874df6620 Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Sat, 22 Aug 2020 16:47:31 -0400 Subject: [PATCH 17/18] partially revert refactor --- cmd/root.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index cccdd93..04b088a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -39,7 +39,12 @@ var ( return err } + // Set up config file location log.Println("Setting up configuration values") + viper.SetConfigName("config") + viper.SetConfigType("yaml") + viper.AddConfigPath(sampleDir) + s, cleanup, err := sample.NewSample(sampleDir, viper.GetStringMapString("cloud_build_subs")) if err != nil { return err @@ -95,11 +100,6 @@ func Execute() error { // init initializes the tool. func init() { // Initialization goes here - // Set up config file location - viper.SetConfigName("config") - viper.SetConfigType("yaml") - //viper.AddConfigPath(sampleDir) - viper.SetEnvPrefix("sst") viper.SetDefault("cloud_build_subs", map[string]string{}) From 1c12fa022888ff90edc77f430ee11867cedd9b0d Mon Sep 17 00:00:00 2001 From: Saketram Durbha Date: Sun, 23 Aug 2020 02:58:54 -0400 Subject: [PATCH 18/18] update comment for cmd.init --- cmd/root.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 04b088a..4f92be4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -97,9 +97,8 @@ func Execute() error { return rootCmd.Execute() } -// init initializes the tool. +// init binds command line flags and environment variables to viper configuration keys. func init() { - // Initialization goes here viper.SetEnvPrefix("sst") viper.SetDefault("cloud_build_subs", map[string]string{})