Skip to content

Commit 1854562

Browse files
michaelhtmmichaelhtm
authored
chore: replace git clone of aws-sdk-go-v2 with HTTP fetch of individual model files (#669)
Description of changes: `ack-generate` previously cloned the entire `aws-sdk-go-v2` repository (hundreds of MB) just to read a single `<service>.json` model file, taking ~2 minutes on a cold cache. This replaces that with a targeted HTTP fetch from GitHub's raw content URL, downloading only the needed model file and caching it locally under `~/.cache/ack-generate/models/<version>/`. Cold generation now completes in ~15 seconds; cached runs similarly in ~15 seconds as well (both run ~3 seconds without `gofmt`) By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
1 parent c80d745 commit 1854562

13 files changed

Lines changed: 232 additions & 234 deletions

File tree

‎cmd/ack-generate/command/apis.go‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
package command
1515

1616
import (
17-
"context"
1817
"fmt"
1918
"io/ioutil"
2019
"path/filepath"
@@ -95,22 +94,18 @@ func generateAPIs(cmd *cobra.Command, args []string) error {
9594
optOutputPath = filepath.Join(optServicesDir, svcAlias)
9695
}
9796

98-
repoStart := time.Now()
99-
ctx, cancel := sdk.ContextWithSigterm(context.Background())
100-
defer cancel()
101-
sdkDirPath, err := sdk.EnsureRepo(ctx, optCacheDir, optRefreshCache, optAWSSDKGoVersion, optOutputPath)
97+
// Load generator config to resolve model name before fetching
98+
cfg, err := setupGenerator(svcAlias)
10299
if err != nil {
103100
return err
104101
}
105-
sdkDir = sdkDirPath
106-
util.Tracef("EnsureRepo: %s\n", time.Since(repoStart))
107102

108103
modelStart := time.Now()
109104
metadata, err := ackmetadata.NewServiceMetadata(optMetadataConfigPath)
110105
if err != nil {
111106
return err
112107
}
113-
m, err := loadModelWithLatestAPIVersion(svcAlias, metadata)
108+
m, err := loadModelWithLatestAPIVersion(svcAlias, metadata, cfg)
114109
if err != nil {
115110
return err
116111
}

‎cmd/ack-generate/command/common.go‎

Lines changed: 49 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
package command
1515

1616
import (
17+
"context"
1718
"fmt"
1819
"sort"
1920
"strings"
@@ -25,36 +26,38 @@ import (
2526
ackgenerate "github.com/aws-controllers-k8s/code-generator/pkg/generate/ack"
2627
ackmetadata "github.com/aws-controllers-k8s/code-generator/pkg/metadata"
2728
ackmodel "github.com/aws-controllers-k8s/code-generator/pkg/model"
29+
"github.com/aws-controllers-k8s/code-generator/pkg/sdk"
2830
acksdk "github.com/aws-controllers-k8s/code-generator/pkg/sdk"
2931
"github.com/aws-controllers-k8s/code-generator/pkg/util"
3032
)
3133

34+
// resolveModelName returns the SDK model name for a service, checking the
35+
// generator config for an override.
36+
func resolveModelName(svcAlias string, cfg ackgenconfig.Config) string {
37+
modelName := strings.ToLower(cfg.SDKNames.Model)
38+
if modelName == "" {
39+
modelName = svcAlias
40+
}
41+
return modelName
42+
}
43+
3244
// loadModelWithLatestAPIVersion finds the AWS SDK for a given service alias and
3345
// creates a new model with the latest API version.
34-
func loadModelWithLatestAPIVersion(svcAlias string, metadata *ackmetadata.ServiceMetadata) (*ackmodel.Model, error) {
46+
func loadModelWithLatestAPIVersion(svcAlias string, metadata *ackmetadata.ServiceMetadata, cfg ackgenconfig.Config) (*ackmodel.Model, error) {
3547
latestAPIVersion, err := getLatestAPIVersion(metadata.APIVersions)
3648
if err != nil {
3749
return nil, err
3850
}
39-
return loadModel(svcAlias, latestAPIVersion, "", ackgenerate.DefaultConfig)
51+
return loadModel(svcAlias, latestAPIVersion, "", cfg)
4052
}
4153

4254
// loadModel finds the AWS SDK for a given service alias and creates a new model
43-
// with the given API version.
44-
func loadModel(svcAlias string, apiVersion string, apiGroup string, defaultCfg ackgenconfig.Config) (*ackmodel.Model, error) {
55+
// with the given API version. The cfg parameter should be pre-loaded by the
56+
// caller so that the model name can be resolved before fetching.
57+
func loadModel(svcAlias string, apiVersion string, apiGroup string, cfg ackgenconfig.Config) (*ackmodel.Model, error) {
4558
totalStart := time.Now()
4659

47-
cfgStart := time.Now()
48-
cfg, err := ackgenconfig.New(optGeneratorConfigPath, defaultCfg)
49-
if err != nil {
50-
return nil, err
51-
}
52-
util.Tracef("config loading: %s\n", time.Since(cfgStart))
53-
54-
modelName := strings.ToLower(cfg.SDKNames.Model)
55-
if modelName == "" {
56-
modelName = svcAlias
57-
}
60+
modelName := resolveModelName(svcAlias, cfg)
5861

5962
apiStart := time.Now()
6063
sdkHelper := acksdk.NewHelper(sdkDir, cfg)
@@ -108,3 +111,34 @@ func getServiceAccountName() (string, error) {
108111

109112
return "", fmt.Errorf("service account name not set")
110113
}
114+
115+
// setupGenerator loads the generator configuration, resolves the SDK version and fetches the
116+
// model file
117+
func setupGenerator(svcAlias string) (ackgenconfig.Config, error) {
118+
// Load generator config to resolve model name before fetching
119+
cfg, err := ackgenconfig.New(optGeneratorConfigPath, ackgenerate.DefaultConfig)
120+
if err != nil {
121+
return cfg, err
122+
}
123+
124+
// Resolve SDK version and fetch the model file
125+
fetchStart := time.Now()
126+
resolvedVersion, err := sdk.GetSDKVersion(optAWSSDKGoVersion, "", optOutputPath)
127+
if err != nil {
128+
return cfg, err
129+
}
130+
resolvedVersion = sdk.EnsureSemverPrefix(resolvedVersion)
131+
132+
modelName := resolveModelName(svcAlias, cfg)
133+
ctx, cancel := sdk.ContextWithSigterm(context.Background())
134+
defer cancel()
135+
basePath, err := sdk.EnsureModel(ctx, optCacheDir, resolvedVersion, modelName)
136+
if err != nil {
137+
return cfg, err
138+
}
139+
sdkDir = basePath
140+
sdkVersion = resolvedVersion
141+
util.Tracef("EnsureModel: %s\n", time.Since(fetchStart))
142+
143+
return cfg, nil
144+
}

‎cmd/ack-generate/command/controller.go‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
package command
1515

1616
import (
17-
"context"
1817
"fmt"
1918
"io/ioutil"
2019
"path/filepath"
@@ -56,22 +55,18 @@ func generateController(cmd *cobra.Command, args []string) error {
5655
optOutputPath = filepath.Join(optServicesDir, svcAlias)
5756
}
5857

59-
repoStart := time.Now()
60-
ctx, cancel := sdk.ContextWithSigterm(context.Background())
61-
defer cancel()
62-
sdkDirPath, err := sdk.EnsureRepo(ctx, optCacheDir, optRefreshCache, optAWSSDKGoVersion, optOutputPath)
58+
// Load generator config to resolve model name before fetching
59+
cfg, err := setupGenerator(svcAlias)
6360
if err != nil {
6461
return err
6562
}
66-
sdkDir = sdkDirPath
67-
util.Tracef("EnsureRepo: %s\n", time.Since(repoStart))
6863

6964
modelStart := time.Now()
7065
metadata, err := ackmetadata.NewServiceMetadata(optMetadataConfigPath)
7166
if err != nil {
7267
return err
7368
}
74-
m, err := loadModelWithLatestAPIVersion(svcAlias, metadata)
69+
m, err := loadModelWithLatestAPIVersion(svcAlias, metadata, cfg)
7570
if err != nil {
7671
return err
7772
}

‎cmd/ack-generate/command/crossplane.go‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"github.com/pkg/errors"
2525
"github.com/spf13/cobra"
2626

27+
ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/config"
2728
cpgenerate "github.com/aws-controllers-k8s/code-generator/pkg/generate/crossplane"
2829
"github.com/aws-controllers-k8s/code-generator/pkg/sdk"
2930
)
@@ -44,21 +45,38 @@ func generateCrossplane(_ *cobra.Command, args []string) error {
4445
return fmt.Errorf("please specify the service alias for the AWS service API to generate")
4546
}
4647

47-
ctx, cancel := sdk.ContextWithSigterm(context.Background())
48-
defer cancel()
49-
sdkDirPath, err := sdk.EnsureRepo(ctx, optCacheDir, optRefreshCache, optAWSSDKGoVersion, optOutputPath)
50-
if err != nil {
51-
return err
52-
}
53-
sdkDir = sdkDirPath
5448
svcAlias := strings.ToLower(args[0])
5549
if optGeneratorConfigPath == "" {
5650
// default generator configuration file path is now: apis/<service>/<generator-config.yaml>
5751
// as this configuration is per API group (per service), and
5852
// resources can exist in multiple versions.
5953
optGeneratorConfigPath = filepath.Join(optOutputPath, "apis", svcAlias, "generator-config.yaml")
6054
}
61-
m, err := loadModel(svcAlias, optGenVersion, "aws.crossplane.io", cpgenerate.DefaultConfig)
55+
56+
// Load generator config to resolve model name before fetching
57+
cfg, err := ackgenconfig.New(optGeneratorConfigPath, cpgenerate.DefaultConfig)
58+
if err != nil {
59+
return err
60+
}
61+
62+
// Resolve SDK version and fetch the model file
63+
resolvedVersion, err := sdk.GetSDKVersion(optAWSSDKGoVersion, "", optOutputPath)
64+
if err != nil {
65+
return err
66+
}
67+
resolvedVersion = sdk.EnsureSemverPrefix(resolvedVersion)
68+
69+
modelName := resolveModelName(svcAlias, cfg)
70+
ctx, cancel := sdk.ContextWithSigterm(context.Background())
71+
defer cancel()
72+
basePath, err := sdk.EnsureModel(ctx, optCacheDir, resolvedVersion, modelName)
73+
if err != nil {
74+
return err
75+
}
76+
sdkDir = basePath
77+
sdkVersion = resolvedVersion
78+
79+
m, err := loadModel(svcAlias, optGenVersion, "aws.crossplane.io", cfg)
6280
if err != nil {
6381
return err
6482
}

‎cmd/ack-generate/command/olm.go‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
package command
1515

1616
import (
17-
"context"
1817
"fmt"
1918
"io/ioutil"
2019
"path/filepath"
@@ -83,19 +82,17 @@ func generateOLMAssets(cmd *cobra.Command, args []string) error {
8382

8483
version := args[1]
8584

86-
// get the generator inputs
87-
ctx, cancel := sdk.ContextWithSigterm(context.Background())
88-
defer cancel()
89-
sdkDirPath, err := sdk.EnsureRepo(ctx, optCacheDir, optRefreshCache, optAWSSDKGoVersion, optOutputPath)
85+
// Load generator config to resolve model name before fetching
86+
cfg, err := setupGenerator(svcAlias)
9087
if err != nil {
9188
return err
9289
}
93-
sdkDir = sdkDirPath
90+
9491
metadata, err := ackmetadata.NewServiceMetadata(optMetadataConfigPath)
9592
if err != nil {
9693
return err
9794
}
98-
m, err := loadModelWithLatestAPIVersion(svcAlias, metadata)
95+
m, err := loadModelWithLatestAPIVersion(svcAlias, metadata, cfg)
9996
if err != nil {
10097
return err
10198
}

‎cmd/ack-generate/command/release.go‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
package command
1515

1616
import (
17-
"context"
1817
"fmt"
1918
"io/ioutil"
2019
"path/filepath"
@@ -59,14 +58,13 @@ func generateRelease(cmd *cobra.Command, args []string) error {
5958
// version supplied hasn't been used (as a Git tag) before...
6059
releaseVersion := strings.ToLower(args[1])
6160

62-
ctx, cancel := sdk.ContextWithSigterm(context.Background())
63-
defer cancel()
64-
sdkDirPath, err := sdk.EnsureRepo(ctx, optCacheDir, optRefreshCache, optAWSSDKGoVersion, optOutputPath)
61+
// Load generator config to resolve model name before fetching
62+
cfg, err := setupGenerator(svcAlias)
6563
if err != nil {
6664
return err
6765
}
68-
sdkDir = sdkDirPath
69-
m, err := loadModel(svcAlias, "", "", ackgenerate.DefaultConfig)
66+
67+
m, err := loadModel(svcAlias, "", "", cfg)
7068
if err != nil {
7169
return err
7270
}

‎cmd/ack-generate/command/root.go‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,14 +32,14 @@ A tool to generate AWS service controller code`
3232
var (
3333
defaultCacheDir string
3434
optCacheDir string
35-
optRefreshCache bool
3635
optAWSSDKGoVersion string
3736
defaultTemplateDirs []string
3837
optTemplateDirs []string
3938
defaultServicesDir string
4039
optServicesDir string
4140
optDryRun bool
4241
sdkDir string
42+
sdkVersion string
4343
optGeneratorConfigPath string
4444
optMetadataConfigPath string
4545
optDocumentationConfigPath string
@@ -107,10 +107,7 @@ func init() {
107107
&optServicesDir, "services-dir", defaultServicesDir, "Path to directory to output service-specific code",
108108
)
109109
rootCmd.PersistentFlags().StringVar(
110-
&optCacheDir, "cache-dir", defaultCacheDir, "Path to directory to store cached files (including clone'd aws-sdk-go-v2 repo)",
111-
)
112-
rootCmd.PersistentFlags().BoolVar(
113-
&optRefreshCache, "refresh-cache", true, "If true, and aws-sdk-go-v2 repo is already cloned, will git pull the latest aws-sdk-go-v2 commit",
110+
&optCacheDir, "cache-dir", defaultCacheDir, "Path to directory to store cached model files",
114111
)
115112
rootCmd.PersistentFlags().StringVar(
116113
&optGeneratorConfigPath, "generator-config-path", "", "Path to file containing instructions for code generation to use",

‎pkg/model/model.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -564,8 +564,7 @@ func (m *Model) RemoveIgnoredOperations(ops *Ops) {
564564

565565
// IsShapeUsedInCRDs returns true if the supplied shape name is a member of amy
566566
// CRD's payloads or those payloads sub-member shapes
567-
func (m *Model) IsShapeUsedInCRDs(shapeName string) bool {
568-
crds, _ := m.GetCRDs()
567+
func (m *Model) IsShapeUsedInCRDs(shapeName string, crds []*CRD) bool {
569568
for _, crd := range crds {
570569
if crd.HasShapeAsMember(shapeName) {
571570
return true
@@ -587,14 +586,19 @@ func (m *Model) GetTypeDefs() ([]*TypeDef, error) {
587586

588587
payloads := m.SDKAPI.GetPayloads()
589588

589+
crds, err := m.GetCRDs()
590+
if err != nil {
591+
return nil, err
592+
}
593+
590594
shapeNames := make([]string, 0, len(m.SDKAPI.API.Shapes))
591595
for shapeName := range m.SDKAPI.API.Shapes {
592596
shapeNames = append(shapeNames, shapeName)
593597
}
594598
sort.Strings(shapeNames)
595599
for _, shapeName := range shapeNames {
596600
shape := m.SDKAPI.API.Shapes[shapeName]
597-
if util.InStrings(shapeName, payloads) && !m.IsShapeUsedInCRDs(shapeName) {
601+
if util.InStrings(shapeName, payloads) && !m.IsShapeUsedInCRDs(shapeName, crds) {
598602
// Payloads are not type defs, unless explicitly used
599603
continue
600604
}
@@ -616,7 +620,7 @@ func (m *Model) GetTypeDefs() ([]*TypeDef, error) {
616620
memberRef := shape.MemberRefs[memberName]
617621
memberNames := names.New(memberName)
618622
memberShape := memberRef.Shape
619-
if !m.IsShapeUsedInCRDs(memberShape.ShapeName) {
623+
if !m.IsShapeUsedInCRDs(memberShape.ShapeName, crds) {
620624
continue
621625
}
622626
gt, err := m.getShapeCleanGoType(memberShape)

‎pkg/model/multiversion/manager.go‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,11 @@
1414
package multiversion
1515

1616
import (
17+
"context"
1718
"errors"
1819
"fmt"
1920
"sort"
21+
"strings"
2022

2123
ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/config"
2224
ackmetadata "github.com/aws-controllers-k8s/code-generator/pkg/metadata"
@@ -84,12 +86,20 @@ func NewAPIVersionManager(
8486
return nil, err
8587
}
8688

87-
sdkAPIHelper := acksdk.NewHelper(sdkCacheDir, cfg)
88-
err = sdkAPIHelper.WithSDKVersion(apiInfo.AWSSDKVersion)
89+
// Resolve the model name and fetch the model file
90+
modelName := strings.ToLower(cfg.SDKNames.Model)
91+
if modelName == "" {
92+
modelName = servicePackageName
93+
}
94+
basePath, err := acksdk.EnsureModel(
95+
context.Background(), sdkCacheDir,
96+
acksdk.EnsureSemverPrefix(apiInfo.AWSSDKVersion), modelName,
97+
)
8998
if err != nil {
9099
return nil, err
91100
}
92101

102+
sdkAPIHelper := acksdk.NewHelper(basePath, cfg)
93103
sdkAPI, err := sdkAPIHelper.API(servicePackageName)
94104
if err != nil {
95105
return nil, err

0 commit comments

Comments
 (0)