From 8aa78c1485f5dafc07eaf82f3bf43cf1ea2b9125 Mon Sep 17 00:00:00 2001 From: Nicolas Degory Date: Sat, 19 Mar 2022 15:38:43 -0700 Subject: [PATCH 1/2] replay incomplete IAM service account creation When the Cloudformation stack created by eksctl for the creation of an IAM role needed for an IAM service account is in the ROLLBACK_COMPLETE status, eksctl should detect it and try to reconcile the desired config and the actual state of the cluster and IAM resources. Issue #4981 --- pkg/cfn/manager/iam.go | 56 +++++++++++++++++++ .../filter/iamserviceaccount_filter.go | 2 +- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/pkg/cfn/manager/iam.go b/pkg/cfn/manager/iam.go index 4487c87407..677baeccae 100644 --- a/pkg/cfn/manager/iam.go +++ b/pkg/cfn/manager/iam.go @@ -3,8 +3,10 @@ package manager import ( "fmt" + "github.com/aws/aws-sdk-go/aws/awserr" cfn "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/kris-nova/logger" + "github.com/pkg/errors" api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" "github.com/weaveworks/eksctl/pkg/cfn/builder" @@ -17,6 +19,53 @@ func (c *StackCollection) makeIAMServiceAccountStackName(namespace, name string) return fmt.Sprintf("eksctl-%s-addon-iamserviceaccount-%s-%s", c.spec.Metadata.Name, namespace, name) } +// stackHasRolledBack alerts of existing stack in rollback status +func (c *StackCollection) stackHasRolledBack(stackName string) (*Stack, error) { + input := &cfn.DescribeStacksInput{ + StackName: &stackName, + } + resp, err := c.cloudformationAPI.DescribeStacks(input) + if err != nil { + aerr, ok := err.(awserr.RequestFailure) + if !ok { + return nil, err + } + if len(aerr.Code()) == 0 { + return nil, err + } + if aerr.Code() != "ValidationError" { + return nil, errors.Wrapf(err, "describing CloudFormation stack %q, code %q", stackName, aerr.Code()) + } + } + if len(resp.Stacks) == 0 { + return nil, nil + } + for _, s := range resp.Stacks { + if *(s.StackStatus) == cfn.StackStatusRollbackComplete { + return s, nil + } + if !c.StackStatusIsNotTransitional(s) { + return nil, errors.Wrapf(err, "stack %q is in a transitional status (%q)", stackName, *(s.StackStatus)) + } + } + return nil, nil +} + +func (c *StackCollection) deleteRolledbackStack(name string) error { + rollbackedStack, err := c.stackHasRolledBack(name) + if err != nil { + return err + } + if rollbackedStack != nil { + logger.Warning("deleting existing rolled back stack %q", name) + err = c.DeleteStackSync(rollbackedStack) + if err != nil { + return err + } + } + return nil +} + // createIAMServiceAccountTask creates the iamserviceaccount in CloudFormation func (c *StackCollection) createIAMServiceAccountTask(errs chan error, spec *api.ClusterIAMServiceAccount, oidc *iamoidc.OpenIDConnectManager) error { name := c.makeIAMServiceAccountStackName(spec.Namespace, spec.Name) @@ -31,6 +80,9 @@ func (c *StackCollection) createIAMServiceAccountTask(errs chan error, spec *api } spec.Tags[api.IAMServiceAccountNameTag] = spec.NameString() + if err := c.deleteRolledbackStack(name); err != nil { + return err + } if err := c.CreateStack(name, stack, spec.Tags, nil, errs); err != nil { logger.Info("an error occurred creating the stack, to cleanup resources, run 'eksctl delete iamserviceaccount --region=%s --name=%s --namespace=%s'", c.spec.Metadata.Region, spec.Name, spec.Namespace) return err @@ -50,6 +102,10 @@ func (c *StackCollection) DescribeIAMServiceAccountStacks() ([]*Stack, error) { if *s.StackStatus == cfn.StackStatusDeleteComplete { continue } + if *s.StackStatus == cfn.StackStatusRollbackComplete { + logger.Warning("unexpected status for stack %v: ROLLBACK_COMPLETE", *s.StackName) + continue + } if GetIAMServiceAccountName(s) != "" { iamServiceAccountStacks = append(iamServiceAccountStacks, s) } diff --git a/pkg/ctl/cmdutils/filter/iamserviceaccount_filter.go b/pkg/ctl/cmdutils/filter/iamserviceaccount_filter.go index 2f7e8ff8b4..c1a388fd37 100644 --- a/pkg/ctl/cmdutils/filter/iamserviceaccount_filter.go +++ b/pkg/ctl/cmdutils/filter/iamserviceaccount_filter.go @@ -78,7 +78,7 @@ func (f *IAMServiceAccountFilter) SetExcludeExistingFilter(stackManager serviceA } // SetDeleteFilter uses stackManager to list existing iamserviceaccount stacks and configures -// the filter to either explictily exluce or include iamserviceaccounts that are missing from given serviceAccounts +// the filter to either explicitly exclude or include iamserviceaccounts that are missing from given serviceAccounts func (f *IAMServiceAccountFilter) SetDeleteFilter(lister serviceAccountLister, includeOnlyMissing bool, cfg *api.ClusterConfig) error { existing, err := lister.ListIAMServiceAccountStacks() if err != nil { From 22bbf7dd2d712a2279405acd58fb9e940359a911 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Mar 2022 20:24:25 +0000 Subject: [PATCH 2/2] Bump github.com/aws/aws-sdk-go from 1.43.17 to 1.43.22 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.43.17 to 1.43.22. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.43.17...v1.43.22) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d98033c0e5..26247b2c99 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ go 1.18 require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/aws/amazon-ec2-instance-selector/v2 v2.0.4-0.20220124212200-2aee60ac608e - github.com/aws/aws-sdk-go v1.43.17 + github.com/aws/aws-sdk-go v1.43.22 github.com/benjamintf1/unmarshalledmatchers v0.0.0-20190408201839-bb1c1f34eaea github.com/blang/semver v3.5.1+incompatible github.com/bxcodec/faker v2.0.1+incompatible diff --git a/go.sum b/go.sum index c0ae1b19c2..18e18e759b 100644 --- a/go.sum +++ b/go.sum @@ -380,8 +380,8 @@ github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zK github.com/aws/aws-sdk-go v1.38.29/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.38.49/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.34/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/aws/aws-sdk-go v1.43.17 h1:jDPBz1UuTxmyRo0eLgaRiro0fiI1zL7lkscqYxoEDLM= -github.com/aws/aws-sdk-go v1.43.17/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.43.22 h1:QY9/1TZB73UDEVQ68sUVJXf/7QUiHZl7zbbLF1wpqlc= +github.com/aws/aws-sdk-go v1.43.22/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2 v1.11.2 h1:SDiCYqxdIYi6HgQfAWRhgdZrdnOuGyLDJVRSWLeHWvs=