Skip to content

Commit af61bc1

Browse files
michaelhtmmichaelhtm
authored
Replace go-git with native git for faster code generation
The code generator was bottlenecked by `go-git` a pure-Go git implementation that is significantly slower than native git for large repositories like aws-sdk-go-v2. `EnsureRepo` was called 3 times per build, each time fetching all remote tags and performing a full `worktree` checkout through `go-git'`s slow object traversal. This made `make build-controller` take 2/3/5+ minutes even with a cached repository. Replaced `go-git` with native `exec.Command("git", ...)` calls, resolved the SDK version before fetching to enable single tag fetch instead of fetching all tags, and added a local tag existence check to skip the network round trip entirely when the tag is already cached. The full ECR controller build now completes in under 10 seconds on repeated runs. Removed go-git and ~10 transitive dependencies from the module. Warm (tag cached, already checked out): | Command | go-git | native git | Speedup | |--------------------------------|---------|------------|-----------| | `ack-generate apis ecr` | 62.3s | 2.3s | **27x** | | `ack-generate controller ecr` | 108.0s | 0.4s | **270x** | Cold (tag not cached, must fetch): | Command | go-git | native git | Speedup | |--------------------------------|---------|------------|-----------| | `ack-generate apis ecr` | 95.1s | 46.2s | **2.1x** | | `ack-generate controller ecr` | 55.2s | 36.2s | **1.5x** | The real optimization isn't the checkout speed (2.3x) - it's skipping the network fetch entirely with HasTag() when the tag already exists locally, which turns a 30-50s operation into a no op. By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
2 parents 5ee9ced + 7de7f41 commit af61bc1

13 files changed

Lines changed: 213 additions & 254 deletions

File tree

‎cmd/ack-generate/command/apis.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"io/ioutil"
2020
"path/filepath"
2121
"strings"
22+
"time"
2223

2324
"github.com/spf13/cobra"
2425

@@ -85,20 +86,26 @@ func saveGeneratedMetadata(cmd *cobra.Command, args []string) error {
8586
// generateAPIs generates the Go files for each resource in the AWS service
8687
// API.
8788
func generateAPIs(cmd *cobra.Command, args []string) error {
89+
cmdStart := time.Now()
8890
if len(args) != 1 {
8991
return fmt.Errorf("please specify the service alias for the AWS service API to generate")
9092
}
9193
svcAlias := strings.ToLower(args[0])
9294
if optOutputPath == "" {
9395
optOutputPath = filepath.Join(optServicesDir, svcAlias)
9496
}
97+
98+
repoStart := time.Now()
9599
ctx, cancel := sdk.ContextWithSigterm(context.Background())
96100
defer cancel()
97101
sdkDirPath, err := sdk.EnsureRepo(ctx, optCacheDir, optRefreshCache, optAWSSDKGoVersion, optOutputPath)
98102
if err != nil {
99103
return err
100104
}
101105
sdkDir = sdkDirPath
106+
util.Tracef("EnsureRepo: %s\n", time.Since(repoStart))
107+
108+
modelStart := time.Now()
102109
metadata, err := ackmetadata.NewServiceMetadata(optMetadataConfigPath)
103110
if err != nil {
104111
return err
@@ -107,15 +114,22 @@ func generateAPIs(cmd *cobra.Command, args []string) error {
107114
if err != nil {
108115
return err
109116
}
117+
util.Tracef("loadModel: %s\n", time.Since(modelStart))
118+
119+
apisStart := time.Now()
110120
ts, err := ackgenerate.APIs(m, optTemplateDirs)
111121
if err != nil {
112122
return err
113123
}
124+
util.Tracef("APIs() template setup: %s\n", time.Since(apisStart))
114125

126+
execStart := time.Now()
115127
if err = ts.Execute(); err != nil {
116128
return err
117129
}
130+
util.Tracef("template execution: %s\n", time.Since(execStart))
118131

132+
writeStart := time.Now()
119133
apisVersionPath = filepath.Join(optOutputPath, "apis", optGenVersion)
120134
for path, contents := range ts.Executed() {
121135
if optDryRun {
@@ -132,5 +146,7 @@ func generateAPIs(cmd *cobra.Command, args []string) error {
132146
return err
133147
}
134148
}
149+
util.Tracef("file writing (%d files): %s\n", len(ts.Executed()), time.Since(writeStart))
150+
util.Tracef("generateAPIs total: %s\n", time.Since(cmdStart))
135151
return nil
136152
}

‎cmd/ack-generate/command/common.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"fmt"
1818
"sort"
1919
"strings"
20+
"time"
2021

2122
k8sversion "k8s.io/apimachinery/pkg/version"
2223

@@ -25,6 +26,7 @@ import (
2526
ackmetadata "github.com/aws-controllers-k8s/code-generator/pkg/metadata"
2627
ackmodel "github.com/aws-controllers-k8s/code-generator/pkg/model"
2728
acksdk "github.com/aws-controllers-k8s/code-generator/pkg/sdk"
29+
"github.com/aws-controllers-k8s/code-generator/pkg/util"
2830
)
2931

3032
// loadModelWithLatestAPIVersion finds the AWS SDK for a given service alias and
@@ -40,21 +42,27 @@ func loadModelWithLatestAPIVersion(svcAlias string, metadata *ackmetadata.Servic
4042
// loadModel finds the AWS SDK for a given service alias and creates a new model
4143
// with the given API version.
4244
func loadModel(svcAlias string, apiVersion string, apiGroup string, defaultCfg ackgenconfig.Config) (*ackmodel.Model, error) {
45+
totalStart := time.Now()
46+
47+
cfgStart := time.Now()
4348
cfg, err := ackgenconfig.New(optGeneratorConfigPath, defaultCfg)
4449
if err != nil {
4550
return nil, err
4651
}
52+
util.Tracef("config loading: %s\n", time.Since(cfgStart))
4753

4854
modelName := strings.ToLower(cfg.SDKNames.Model)
4955
if modelName == "" {
5056
modelName = svcAlias
5157
}
5258

59+
apiStart := time.Now()
5360
sdkHelper := acksdk.NewHelper(sdkDir, cfg)
5461
sdkAPI, err := sdkHelper.API(modelName)
5562
if err != nil {
5663
return nil, err
5764
}
65+
util.Tracef("SDK API loading (model=%s): %s\n", modelName, time.Since(apiStart))
5866

5967
if apiGroup != "" {
6068
sdkAPI.APIGroupSuffix = apiGroup
@@ -65,12 +73,15 @@ func loadModel(svcAlias string, apiVersion string, apiGroup string, defaultCfg a
6573
return nil, err
6674
}
6775

76+
modelStart := time.Now()
6877
m, err := ackmodel.New(
6978
sdkAPI, svcAlias, apiVersion, cfg, docCfg,
7079
)
7180
if err != nil {
7281
return nil, err
7382
}
83+
util.Tracef("model.New(): %s\n", time.Since(modelStart))
84+
util.Tracef("loadModel total: %s\n", time.Since(totalStart))
7485
return m, nil
7586
}
7687

‎cmd/ack-generate/command/controller.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,14 @@ import (
1919
"io/ioutil"
2020
"path/filepath"
2121
"strings"
22+
"time"
2223

2324
"github.com/spf13/cobra"
2425

2526
ackgenerate "github.com/aws-controllers-k8s/code-generator/pkg/generate/ack"
2627
ackmetadata "github.com/aws-controllers-k8s/code-generator/pkg/metadata"
2728
"github.com/aws-controllers-k8s/code-generator/pkg/sdk"
29+
"github.com/aws-controllers-k8s/code-generator/pkg/util"
2830
)
2931

3032
var (
@@ -45,6 +47,7 @@ func init() {
4547

4648
// generateController generates the Go files for a service controller
4749
func generateController(cmd *cobra.Command, args []string) error {
50+
cmdStart := time.Now()
4851
if len(args) != 1 {
4952
return fmt.Errorf("please specify the service alias for the AWS service API to generate")
5053
}
@@ -53,13 +56,17 @@ func generateController(cmd *cobra.Command, args []string) error {
5356
optOutputPath = filepath.Join(optServicesDir, svcAlias)
5457
}
5558

59+
repoStart := time.Now()
5660
ctx, cancel := sdk.ContextWithSigterm(context.Background())
5761
defer cancel()
5862
sdkDirPath, err := sdk.EnsureRepo(ctx, optCacheDir, optRefreshCache, optAWSSDKGoVersion, optOutputPath)
5963
if err != nil {
6064
return err
6165
}
6266
sdkDir = sdkDirPath
67+
util.Tracef("EnsureRepo: %s\n", time.Since(repoStart))
68+
69+
modelStart := time.Now()
6370
metadata, err := ackmetadata.NewServiceMetadata(optMetadataConfigPath)
6471
if err != nil {
6572
return err
@@ -68,19 +75,27 @@ func generateController(cmd *cobra.Command, args []string) error {
6875
if err != nil {
6976
return err
7077
}
78+
util.Tracef("loadModel: %s\n", time.Since(modelStart))
79+
7180
serviceAccountName, err := getServiceAccountName()
7281
if err != nil {
7382
return err
7483
}
84+
85+
ctrlStart := time.Now()
7586
ts, err := ackgenerate.Controller(m, optTemplateDirs, serviceAccountName)
7687
if err != nil {
7788
return err
7889
}
90+
util.Tracef("Controller() template setup: %s\n", time.Since(ctrlStart))
7991

92+
execStart := time.Now()
8093
if err = ts.Execute(); err != nil {
8194
return err
8295
}
96+
util.Tracef("template execution: %s\n", time.Since(execStart))
8397

98+
writeStart := time.Now()
8499
for path, contents := range ts.Executed() {
85100
if optDryRun {
86101
fmt.Printf("============================= %s ======================================\n", path)
@@ -96,5 +111,7 @@ func generateController(cmd *cobra.Command, args []string) error {
96111
return err
97112
}
98113
}
114+
util.Tracef("file writing (%d files): %s\n", len(ts.Executed()), time.Since(writeStart))
115+
util.Tracef("generateController total: %s\n", time.Since(cmdStart))
99116
return nil
100117
}

‎go.mod‎

Lines changed: 9 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -11,23 +11,19 @@ require (
1111
// pin to v0.1.1 due to release problem with v0.1.2
1212
github.com/gertd/go-pluralize v0.1.1
1313
github.com/ghodss/yaml v1.0.0
14-
github.com/go-git/go-git/v5 v5.11.0
1514
github.com/go-logr/logr v1.4.2
1615
github.com/iancoleman/strcase v0.2.0
1716
github.com/operator-framework/api v0.6.0
1817
github.com/pkg/errors v0.9.1
1918
github.com/samber/lo v1.37.0
2019
github.com/spf13/cobra v1.8.1
21-
github.com/stretchr/testify v1.9.0
22-
golang.org/x/mod v0.24.0
20+
github.com/stretchr/testify v1.10.0
21+
golang.org/x/mod v0.29.0
2322
k8s.io/apimachinery v0.32.1
2423
sigs.k8s.io/controller-runtime v0.20.4
2524
)
2625

2726
require (
28-
dario.cat/mergo v1.0.0 // indirect
29-
github.com/Microsoft/go-winio v0.6.1 // indirect
30-
github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
3127
github.com/aws/aws-sdk-go-v2/config v1.28.6 // indirect
3228
github.com/aws/aws-sdk-go-v2/credentials v1.17.47 // indirect
3329
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect
@@ -43,67 +39,52 @@ require (
4339
github.com/beorn7/perks v1.0.1 // indirect
4440
github.com/blang/semver/v4 v4.0.0 // indirect
4541
github.com/cespare/xxhash/v2 v2.3.0 // indirect
46-
github.com/cloudflare/circl v1.3.7 // indirect
47-
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
4842
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
4943
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
50-
github.com/emirpasic/gods v1.18.1 // indirect
5144
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
5245
github.com/fsnotify/fsnotify v1.7.0 // indirect
5346
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
54-
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
55-
github.com/go-git/go-billy/v5 v5.5.0 // indirect
5647
github.com/go-logr/zapr v1.3.0 // indirect
5748
github.com/go-openapi/jsonpointer v0.21.0 // indirect
5849
github.com/go-openapi/jsonreference v0.21.0 // indirect
5950
github.com/go-openapi/swag v0.23.0 // indirect
6051
github.com/gogo/protobuf v1.3.2 // indirect
61-
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
6252
github.com/golang/protobuf v1.5.4 // indirect
6353
github.com/google/gnostic-models v0.6.8 // indirect
64-
github.com/google/go-cmp v0.6.0 // indirect
54+
github.com/google/go-cmp v0.7.0 // indirect
6555
github.com/google/gofuzz v1.2.0 // indirect
6656
github.com/google/uuid v1.6.0 // indirect
6757
github.com/inconshreveable/mousetrap v1.1.0 // indirect
6858
github.com/jaypipes/envutil v1.0.0 // indirect
69-
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
7059
github.com/jmespath/go-jmespath v0.4.0 // indirect
7160
github.com/josharian/intern v1.0.0 // indirect
7261
github.com/json-iterator/go v1.1.12 // indirect
73-
github.com/kevinburke/ssh_config v1.2.0 // indirect
7462
github.com/kro-run/kro v0.2.2
7563
github.com/mailru/easyjson v0.7.7 // indirect
7664
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
7765
github.com/modern-go/reflect2 v1.0.2 // indirect
7866
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
79-
github.com/pjbgf/sha1cd v0.3.0 // indirect
8067
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
8168
github.com/prometheus/client_golang v1.19.1 // indirect
8269
github.com/prometheus/client_model v0.6.1 // indirect
8370
github.com/prometheus/common v0.55.0 // indirect
8471
github.com/prometheus/procfs v0.15.1 // indirect
85-
github.com/sergi/go-diff v1.1.0 // indirect
8672
github.com/sirupsen/logrus v1.9.3 // indirect
87-
github.com/skeema/knownhosts v1.2.1 // indirect
8873
github.com/spf13/pflag v1.0.5 // indirect
8974
github.com/x448/float16 v0.8.4 // indirect
90-
github.com/xanzy/ssh-agent v0.3.3 // indirect
9175
go.uber.org/multierr v1.11.0 // indirect
9276
go.uber.org/zap v1.27.0 // indirect
93-
golang.org/x/crypto v0.36.0 // indirect
9477
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
95-
golang.org/x/net v0.38.0 // indirect
78+
golang.org/x/net v0.47.0 // indirect
9679
golang.org/x/oauth2 v0.27.0 // indirect
97-
golang.org/x/sync v0.12.0 // indirect
98-
golang.org/x/sys v0.31.0 // indirect
99-
golang.org/x/term v0.30.0 // indirect
100-
golang.org/x/text v0.23.0 // indirect
80+
golang.org/x/sync v0.18.0 // indirect
81+
golang.org/x/sys v0.38.0 // indirect
82+
golang.org/x/term v0.37.0 // indirect
83+
golang.org/x/text v0.31.0 // indirect
10184
golang.org/x/time v0.7.0 // indirect
102-
golang.org/x/tools v0.31.0 // indirect
10385
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
10486
google.golang.org/protobuf v1.35.1 // indirect
10587
gopkg.in/inf.v0 v0.9.1 // indirect
106-
gopkg.in/warnings.v0 v0.1.2 // indirect
10788
gopkg.in/yaml.v2 v2.4.0 // indirect
10889
gopkg.in/yaml.v3 v3.0.1 // indirect
10990
k8s.io/api v0.32.1 // indirect
@@ -119,5 +100,6 @@ require (
119100

120101
require (
121102
github.com/google/btree v1.1.3 // indirect
103+
github.com/rogpeppe/go-internal v1.14.1 // indirect
122104
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
123105
)

0 commit comments

Comments
 (0)