From 5ce765fca56b614e089a388b8a1bbe24b6b689ed Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Fri, 14 Feb 2020 14:29:31 +0100 Subject: [PATCH 1/7] Allow sharing the Go cache directory for crossbuilds Signed-off-by: Simon Pasquier --- .circleci/config.yml | 8 +++++++- .promu.yml | 3 +-- cmd/crossbuild.go | 24 ++++++++++++++++++------ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fdc0b52b..3c2ec55b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,7 +30,13 @@ jobs: - setup_remote_docker - checkout - run: make promu - - run: promu crossbuild -v + - restore_cache: + key: v1-promu-config-{{ checksum ".promu.yml" }} + - run: promu crossbuild -v --gocache-path /home/circleci/.cache/go-build + - save_cache: + key: v1-promu-config-{{ checksum ".promu.yml" }} + paths: + - /home/circleci/.cache/go-build - persist_to_workspace: root: . paths: diff --git a/.promu.yml b/.promu.yml index 9e6c9f8c..45d18272 100644 --- a/.promu.yml +++ b/.promu.yml @@ -5,7 +5,7 @@ go: repository: path: github.com/prometheus/promu build: - flags: -mod=vendor -a -tags 'netgo static_build' + flags: -mod=vendor -tags 'netgo static_build' ldflags: | -s -X github.com/prometheus/common/version.Version={{.Version}} @@ -17,4 +17,3 @@ tarball: files: - LICENSE - NOTICE - diff --git a/cmd/crossbuild.go b/cmd/crossbuild.go index 37f99481..c91864c8 100644 --- a/cmd/crossbuild.go +++ b/cmd/crossbuild.go @@ -78,6 +78,7 @@ var ( platformsFlagSet = true return nil }).Strings() + goCachePath = crossbuildcmd.Flag("gocache-path", "Path to a Go cache directory on the local machine").String() // kingpin doesn't currently support using the crossbuild command and the // crossbuild tarball subcommand at the same time, so we treat the // tarball subcommand as an optional arg @@ -85,7 +86,7 @@ var ( ) func runCrossbuild() { - //Check required configuration + // Check required configuration. if len(strings.TrimSpace(config.Repository.Path)) == 0 { log.Fatalf("missing required '%s' configuration", "repository.path") } @@ -158,7 +159,7 @@ func runCrossbuild() { allPlatforms = append(allPlatforms, s390xPlatforms[:]...) pg := &platformGroup{"base", dockerBaseBuilderImage, allPlatforms} - if err := pg.Build(repoPath); err != nil { + if err := pg.Build(repoPath, *goCachePath); err != nil { fatal(errors.Wrapf(err, "The %s builder docker image exited unexpectedly", pg.Name)) } } else { @@ -169,7 +170,7 @@ func runCrossbuild() { {"MIPS", dockerMIPSBuilderImage, mipsPlatforms}, {"s390x", dockerS390XBuilderImage, s390xPlatforms}, } { - if err := pg.Build(repoPath); err != nil { + if err := pg.Build(repoPath, *goCachePath); err != nil { fatal(errors.Wrapf(err, "The %s builder docker image exited unexpectedly", pg.Name)) } } @@ -182,7 +183,7 @@ type platformGroup struct { Platforms []string } -func (pg platformGroup) Build(repoPath string) error { +func (pg platformGroup) Build(repoPath string, goCachePath string) error { platformsParam := strings.Join(pg.Platforms[:], " ") if len(platformsParam) == 0 { return nil @@ -196,11 +197,22 @@ func (pg platformGroup) Build(repoPath string) error { } ctrName := "promu-crossbuild-" + pg.Name + strconv.FormatInt(time.Now().Unix(), 10) - err = sh.RunCommand("docker", "create", "-t", + args := []string{ + "create", "-t", "--name", ctrName, + } + if goCachePath != "" { + args = append(args, + "--env", "GOCACHE=/root/.cache/go-build", + "-v", fmt.Sprintf("%s:/root/.cache/go-build", goCachePath), + ) + } + args = append(args, pg.DockerImage, "-i", repoPath, - "-p", platformsParam) + "-p", platformsParam, + ) + err = sh.RunCommand("docker", args...) if err != nil { return err } From 7c768cdf1cd04f79005366c7d7199eebe80bf74e Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Fri, 14 Feb 2020 14:36:14 +0100 Subject: [PATCH 2/7] Build promu from sources in the CI build job Signed-off-by: Simon Pasquier --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3c2ec55b..fe1246d8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,7 @@ jobs: steps: - setup_remote_docker - checkout - - run: make promu + - run: make build - restore_cache: key: v1-promu-config-{{ checksum ".promu.yml" }} - run: promu crossbuild -v --gocache-path /home/circleci/.cache/go-build From cab1a56ad02a4797df0eca832665d8ac0e436412 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Fri, 14 Feb 2020 14:52:30 +0100 Subject: [PATCH 3/7] Use a constant for the container's cache directory Signed-off-by: Simon Pasquier --- cmd/crossbuild.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/crossbuild.go b/cmd/crossbuild.go index c91864c8..d1647baa 100644 --- a/cmd/crossbuild.go +++ b/cmd/crossbuild.go @@ -81,7 +81,7 @@ var ( goCachePath = crossbuildcmd.Flag("gocache-path", "Path to a Go cache directory on the local machine").String() // kingpin doesn't currently support using the crossbuild command and the // crossbuild tarball subcommand at the same time, so we treat the - // tarball subcommand as an optional arg + // tarball subcommand as an optional arg. tarballsSubcommand = crossbuildcmd.Arg("tarballs", "Optionally pass the string \"tarballs\" from cross-built binaries").String() ) @@ -150,7 +150,7 @@ func runCrossbuild() { } if !cgo { - // In non-CGO, use the base image without any crossbuild toolchain + // In non-CGO, use the base image without any crossbuild toolchain. var allPlatforms []string allPlatforms = append(allPlatforms, mainPlatforms[:]...) allPlatforms = append(allPlatforms, armPlatforms[:]...) @@ -183,6 +183,8 @@ type platformGroup struct { Platforms []string } +const containerGoCacheDir = "/root/.cache/go-build" + func (pg platformGroup) Build(repoPath string, goCachePath string) error { platformsParam := strings.Join(pg.Platforms[:], " ") if len(platformsParam) == 0 { @@ -203,8 +205,8 @@ func (pg platformGroup) Build(repoPath string, goCachePath string) error { } if goCachePath != "" { args = append(args, - "--env", "GOCACHE=/root/.cache/go-build", - "-v", fmt.Sprintf("%s:/root/.cache/go-build", goCachePath), + "--env", fmt.Sprintf("GOCACHE=%s", containerGoCacheDir), + "-v", fmt.Sprintf("%s:%s", goCachePath, containerGoCacheDir), ) } args = append(args, From 1958603c5fa743b8e74876b3a35a1897b0df0dcf Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Fri, 14 Feb 2020 15:42:08 +0100 Subject: [PATCH 4/7] use go.mod as the cache's key Signed-off-by: Simon Pasquier --- .circleci/config.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fe1246d8..03115790 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,10 +31,14 @@ jobs: - checkout - run: make build - restore_cache: - key: v1-promu-config-{{ checksum ".promu.yml" }} + keys: + - v0-gocache-{{ checksum "go.mod" }} + - v0-gocache- + - run: mkdir -p /home/circleci/.cache/go-build - run: promu crossbuild -v --gocache-path /home/circleci/.cache/go-build + - run: du -sh /home/circleci/.cache/go-build - save_cache: - key: v1-promu-config-{{ checksum ".promu.yml" }} + key: v0-gocache-{{ checksum "go.mod" }} paths: - /home/circleci/.cache/go-build - persist_to_workspace: From a040f46163ce60d351cbaf40cd53a521fe256e6b Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Fri, 14 Feb 2020 16:26:21 +0100 Subject: [PATCH 5/7] Use machine executor Signed-off-by: Simon Pasquier --- .circleci/config.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 03115790..c28806ad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,9 @@ --- version: 2.1 +orbs: + go: circleci/go@0.2.0 + executors: # Whenever the Go version is updated here, .promu.yml should also be updated. golang: @@ -23,22 +26,23 @@ jobs: - run: git diff --exit-code build: - executor: golang + machine: true working_directory: /home/circleci/.go_workspace/src/github.com/prometheus/promu steps: - - setup_remote_docker - checkout + - go/install: + version: 1.13.8 - run: make build - restore_cache: keys: - - v0-gocache-{{ checksum "go.mod" }} - - v0-gocache- + - v01-gocache-{{ checksum "go.mod" }} + - v01-gocache- - run: mkdir -p /home/circleci/.cache/go-build - run: promu crossbuild -v --gocache-path /home/circleci/.cache/go-build - run: du -sh /home/circleci/.cache/go-build - save_cache: - key: v0-gocache-{{ checksum "go.mod" }} + key: v01-gocache-{{ checksum "go.mod" }} paths: - /home/circleci/.cache/go-build - persist_to_workspace: From f42b9d524ff7adadf837ba441b806703ac8ec447 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Fri, 14 Feb 2020 16:38:24 +0100 Subject: [PATCH 6/7] Update machine image Signed-off-by: Simon Pasquier --- .circleci/config.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c28806ad..8fa2db57 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,9 +1,6 @@ --- version: 2.1 -orbs: - go: circleci/go@0.2.0 - executors: # Whenever the Go version is updated here, .promu.yml should also be updated. golang: @@ -26,13 +23,12 @@ jobs: - run: git diff --exit-code build: - machine: true + machine: + image: ubuntu-1604:201903-01 working_directory: /home/circleci/.go_workspace/src/github.com/prometheus/promu steps: - checkout - - go/install: - version: 1.13.8 - run: make build - restore_cache: keys: From b3aefe63d366ecdd1fe4690968dbd6f05e8d21e8 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Fri, 14 Feb 2020 16:40:18 +0100 Subject: [PATCH 7/7] Just run go install Signed-off-by: Simon Pasquier --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8fa2db57..9eb34059 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,7 @@ jobs: steps: - checkout - - run: make build + - run: go install . - restore_cache: keys: - v01-gocache-{{ checksum "go.mod" }}