From 259d2d9835ec4beb5b34a60cb626ce86c42b7b59 Mon Sep 17 00:00:00 2001 From: Christophe MORIO Date: Tue, 4 Sep 2018 12:21:12 +0200 Subject: [PATCH 01/62] Fix elb arn namespace : "elasticloadbalancing" instead of "elb" --- skew/resources/aws/elb.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/skew/resources/aws/elb.py b/skew/resources/aws/elb.py index 1938d7c..527fc22 100644 --- a/skew/resources/aws/elb.py +++ b/skew/resources/aws/elb.py @@ -31,3 +31,10 @@ class Meta(object): dimension = 'LoadBalancerName' tags_spec = ('describe_tags', 'TagDescriptions[].Tags[]', 'LoadBalancerNames', 'id') + + @property + def arn(self): + return 'arn:aws:elasticloadbalancing:%s:%s:%s/%s' % ( + self._client.region_name, + self._client.account_id, + self.resourcetype, self.id) From 0e8521bce5c4b635c4ac08b6ba0626774dd11195 Mon Sep 17 00:00:00 2001 From: Christophe MORIO Date: Tue, 4 Sep 2018 12:22:25 +0200 Subject: [PATCH 02/62] Add support for 8 resources - aws.acm.certificate - aws.ecs.cluster - aws.ecs.task-definition - aws.efs.filesystem - aws.elbv2.loadbalancer - aws.elbv2.targetgroup - aws.ses.identity - aws.support.check --- skew/resources/__init__.py | 10 ++++- skew/resources/aws/acm.py | 38 ++++++++++++++++++ skew/resources/aws/ecs.py | 76 +++++++++++++++++++++++++++++++++++ skew/resources/aws/efs.py | 47 ++++++++++++++++++++++ skew/resources/aws/elbv2.py | 70 ++++++++++++++++++++++++++++++++ skew/resources/aws/ses.py | 44 ++++++++++++++++++++ skew/resources/aws/support.py | 37 +++++++++++++++++ 7 files changed, 321 insertions(+), 1 deletion(-) create mode 100644 skew/resources/aws/acm.py create mode 100644 skew/resources/aws/ecs.py create mode 100644 skew/resources/aws/efs.py create mode 100644 skew/resources/aws/elbv2.py create mode 100644 skew/resources/aws/ses.py create mode 100644 skew/resources/aws/support.py diff --git a/skew/resources/__init__.py b/skew/resources/__init__.py index 49d1dd4..e145983 100644 --- a/skew/resources/__init__.py +++ b/skew/resources/__init__.py @@ -17,6 +17,7 @@ # Maps resources names as they appear in ARN's to the path name # of the Python class representing that resource. ResourceTypes = { + 'aws.acm.certificate': 'aws.acm.Certificate', 'aws.apigateway.restapis': 'aws.apigateway.RestAPI', 'aws.autoscaling.autoScalingGroup': 'aws.autoscaling.AutoScalingGroup', 'aws.autoscaling.launchConfigurationName': 'aws.autoscaling.LaunchConfiguration', @@ -37,12 +38,17 @@ 'aws.ec2.vpc': 'aws.ec2.Vpc', 'aws.ec2.vpc-peering-connection': 'aws.ec2.VpcPeeringConnection', 'aws.ec2.subnet': 'aws.ec2.Subnet', + 'aws.ecs.cluster': 'aws.ecs.Cluster', + 'aws.ecs.task-definition': 'aws.ecs.TaskDefinition', + 'aws.efs.filesystem': 'aws.efs.Filesystem', 'aws.elasticache.cluster': 'aws.elasticache.Cluster', 'aws.elasticache.subnet-group': 'aws.elasticache.SubnetGroup', 'aws.elasticache.snapshot': 'aws.elasticache.Snapshot', 'aws.elasticbeanstalk.application': 'aws.elasticbeanstalk.Application', 'aws.elasticbeanstalk.environment': 'aws.elasticbeanstalk.Environment', 'aws.elb.loadbalancer': 'aws.elb.LoadBalancer', + 'aws.elbv2.loadbalancer': 'aws.elbv2.LoadBalancer', + 'aws.elbv2.targetgroup': 'aws.elbv2.TargetGroup', 'aws.es.domain': 'aws.es.ElasticsearchDomain', 'aws.firehose.deliverystream': 'aws.firehose.DeliveryStream', 'aws.iam.group': 'aws.iam.Group', @@ -60,8 +66,10 @@ 'aws.route53.healthcheck': 'aws.route53.HealthCheck', 'aws.s3.bucket': 'aws.s3.Bucket', 'aws.sqs.queue': 'aws.sqs.Queue', + 'aws.ses.identity': 'aws.ses.Identity', 'aws.sns.subscription': 'aws.sns.Subscription', - 'aws.sns.topic': 'aws.sns.Topic' + 'aws.sns.topic': 'aws.sns.Topic', + 'aws.support.check': 'aws.support.Check' } diff --git a/skew/resources/aws/acm.py b/skew/resources/aws/acm.py new file mode 100644 index 0000000..e6da847 --- /dev/null +++ b/skew/resources/aws/acm.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Certificate(AWSResource): + + class Meta(object): + service = 'acm' + type = 'certificate' + enum_spec = ('list_certificates', 'CertificateSummaryList', None) + detail_spec = None + id = 'CertificateArn' + tags_spec = ('list_tags_for_certificate', 'Tags[]', + 'CertificateArn', 'id') + filter_name = None + name = 'DomainName' + date = None + dimension = None diff --git a/skew/resources/aws/ecs.py b/skew/resources/aws/ecs.py new file mode 100644 index 0000000..91663dd --- /dev/null +++ b/skew/resources/aws/ecs.py @@ -0,0 +1,76 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Cluster(AWSResource): + + class Meta(object): + service = 'ecs' + type = 'cluster' + enum_spec = ('list_clusters', 'clusterArns', None) + detail_spec = ('describe_clusters', 'clusters', 'clusters[0]') + id = None + + + filter_name = None + name = 'clusterName' + date = None + dimension = None + + @property + def arn(self): + return self.data['clusterArn'] + + def __init__(self, client, data, query=None): + super(Cluster, self).__init__(client, data, query) + self._id = data + detail_op, param_name, detail_path = self.Meta.detail_spec + params = {param_name: [self.id]} + data = client.call(detail_op, **params) + self.data = jmespath.search(detail_path, data) + + +class TaskDefinition(AWSResource): + + class Meta(object): + service = 'ecs' + type = 'task-definition' + enum_spec = ('list_task_definitions', 'taskDefinitionArns', None) + detail_spec = ('describe_task_definition', 'taskDefinition', 'taskDefinition') + id = None + name = None + filter_name = None + date = None + dimension = None + + def __init__(self, client, data, query=None): + super(TaskDefinition, self).__init__(client, data, query) + self._id = data + detail_op, param_name, detail_path = self.Meta.detail_spec + params = {param_name: self.id} + data = client.call(detail_op, **params) + self.data = jmespath.search(detail_path, data) + + @property + def arn(self): + return self.data['taskDefinitionArn'] \ No newline at end of file diff --git a/skew/resources/aws/efs.py b/skew/resources/aws/efs.py new file mode 100644 index 0000000..e5c21bd --- /dev/null +++ b/skew/resources/aws/efs.py @@ -0,0 +1,47 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Filesystem(AWSResource): + + class Meta(object): + service = 'efs' + type = 'filesystem' + enum_spec = ('describe_file_systems', 'FileSystems', None) + detail_spec = None + id = 'FileSystemId' + tags_spec = ('describe_tags', 'Tags[]', + 'FileSystemId', 'id') + filter_name = None + name = 'Name' + date = 'CreationTime' + dimension = None + + @property + def arn(self): + # arn:aws:elasticfilesystem:us-east-1:123456789012:file-system-id/fs12345678 + return 'arn:aws:%s:%s:%s:%s/%s' % ( + 'elasticfilesystem', + self._client.region_name, + self._client.account_id, + 'file-system-id', self.id) diff --git a/skew/resources/aws/elbv2.py b/skew/resources/aws/elbv2.py new file mode 100644 index 0000000..efebe24 --- /dev/null +++ b/skew/resources/aws/elbv2.py @@ -0,0 +1,70 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import jmespath + +from skew.resources.aws import AWSResource + + +class LoadBalancer(AWSResource): + + class Meta(object): + service = 'elbv2' + type = 'loadbalancer' + enum_spec = ('describe_load_balancers', + 'LoadBalancers', None) + # detail_spec = None + detail_spec = ('describe_listeners', 'LoadBalancerArn', 'Listeners') + id = 'LoadBalancerArn' + filter_name = 'Names' + filter_type = 'list' + name = 'DNSName' + date = 'CreatedTime' + dimension = None + tags_spec = ('describe_tags', 'TagDescriptions[].Tags[]', + 'LoadBalancerNames', 'id') + + @property + def arn(self): + return self.data['LoadBalancerArn'] + + def __init__(self, client, data, query=None): + super(LoadBalancer, self).__init__(client, data, query) + self._id = data + detail_op, param_name, detail_path = self.Meta.detail_spec + params = {param_name: self.data['LoadBalancerArn']} + data = client.call(detail_op, **params) + self.data['Listeners'] = jmespath.search(detail_path, data) + + +class TargetGroup(AWSResource): + + class Meta(object): + service = 'elbv2' + type = 'targetgroup' + enum_spec = ('describe_target_groups', + 'TargetGroups', None) + detail_spec = None + id = 'TargetGroupArn' + filter_name = 'Names' + filter_type = 'list' + name = 'TargetGroupName' + date = 'CreatedTime' + dimension = 'LoadBalancerName' + tags_spec = ('describe_tags', 'TagDescriptions[].Tags[]', + 'LoadBalancerNames', 'id') + + @property + def arn(self): + return self.data['TargetGroupArn'] diff --git a/skew/resources/aws/ses.py b/skew/resources/aws/ses.py new file mode 100644 index 0000000..90a575a --- /dev/null +++ b/skew/resources/aws/ses.py @@ -0,0 +1,44 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Identity(AWSResource): + + class Meta(object): + service = 'ses' + type = 'identity' + enum_spec = ('list_identities', 'Identities', None) + detail_spec = ('describe_table', 'TableName', 'Table') + id = 'Identity' + tags_spec = None + filter_name = None + name = 'IdentityName' + date = None + dimension = 'IdentityName' + + @property + def arn(self): + return 'arn:aws:%s:%s:%s:%s/%s' % ( + self._client.service_name, + self._client.region_name, + self._client.account_id, self.resourcetype, self.data) diff --git a/skew/resources/aws/support.py b/skew/resources/aws/support.py new file mode 100644 index 0000000..61733e6 --- /dev/null +++ b/skew/resources/aws/support.py @@ -0,0 +1,37 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Check(AWSResource): + + class Meta(object): + service = 'support' + type = 'check' + enum_spec = ('describe_trusted_advisor_checks', 'checks', None) + detail_spec = None + id = 'id' + tags_spec = None + filter_name = None + name = 'name' + date = None + dimension = 'IdentityName' From 828768ebee274e526cbe1463e68432a96a351db1 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Thu, 13 Sep 2018 14:55:02 +0200 Subject: [PATCH 03/62] Return specific ARN --- skew/resources/aws/acm.py | 4 ++++ skew/resources/aws/route53.py | 4 ++++ skew/resources/aws/s3.py | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/skew/resources/aws/acm.py b/skew/resources/aws/acm.py index e6da847..6e9b0e9 100644 --- a/skew/resources/aws/acm.py +++ b/skew/resources/aws/acm.py @@ -36,3 +36,7 @@ class Meta(object): name = 'DomainName' date = None dimension = None + + @property + def arn(self): + return self.data['CertificateArn'] diff --git a/skew/resources/aws/route53.py b/skew/resources/aws/route53.py index 5c07a94..2874ef3 100644 --- a/skew/resources/aws/route53.py +++ b/skew/resources/aws/route53.py @@ -42,6 +42,10 @@ class Meta(object): def id(self): return self._id.split('/')[-1] + @property + def arn(self): + return 'arn:aws:route:::hostedzone/{}'.format(self.id) + class HealthCheck(Route53Resource): diff --git a/skew/resources/aws/s3.py b/skew/resources/aws/s3.py index 98a2b25..3057752 100644 --- a/skew/resources/aws/s3.py +++ b/skew/resources/aws/s3.py @@ -72,3 +72,7 @@ def __iter__(self): self._keys = jmespath.search(detail_path, data) for key in self._keys: yield key + + @property + def arn(self): + return 'arn:aws:s3:::{}'.format(self.data['Name']) From 95dd06f5d15b9eabe54b8d9525c69bb5e663a728 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Thu, 13 Sep 2018 14:56:33 +0200 Subject: [PATCH 04/62] Sort a list to get consistent outputs --- skew/resources/aws/autoscaling.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/skew/resources/aws/autoscaling.py b/skew/resources/aws/autoscaling.py index 0d84ee2..78d5b6c 100644 --- a/skew/resources/aws/autoscaling.py +++ b/skew/resources/aws/autoscaling.py @@ -32,6 +32,8 @@ class Meta(object): filter_type = 'list' def __init__(self, client, data, query=None): + # Always save the list in the same order to avoid false changes detection + data['EnabledMetrics'].sort(key=lambda item:item['Metric']) super(AutoScalingGroup, self).__init__(client, data, query) self._arn_query = jmespath.compile('AutoScalingGroupARN') From 02af22771a824d16385a7af7765d8f14cf3cfc1a Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 1 Oct 2018 10:23:19 +0200 Subject: [PATCH 05/62] Add set tag feature for ACM, ASG and CloudFront --- skew/resources/aws/acm.py | 8 ++++++++ skew/resources/aws/autoscaling.py | 12 ++++++++++++ skew/resources/aws/cloudfront.py | 8 ++++++++ skew/resources/resource.py | 4 ++++ 4 files changed, 32 insertions(+) diff --git a/skew/resources/aws/acm.py b/skew/resources/aws/acm.py index 6e9b0e9..e754739 100644 --- a/skew/resources/aws/acm.py +++ b/skew/resources/aws/acm.py @@ -17,6 +17,7 @@ import jmespath from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient LOG = logging.getLogger(__name__) @@ -40,3 +41,10 @@ class Meta(object): @property def arn(self): return self.data['CertificateArn'] + + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + tags_list = [dict(Key=k, Value=str(v)) for k, v in tags.items()] + return client.call('add_tags_to_certificate', CertificateArn=arn, Tags=tags_list) diff --git a/skew/resources/aws/autoscaling.py b/skew/resources/aws/autoscaling.py index 78d5b6c..54a191a 100644 --- a/skew/resources/aws/autoscaling.py +++ b/skew/resources/aws/autoscaling.py @@ -15,6 +15,7 @@ import jmespath from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient class AutoScalingGroup(AWSResource): @@ -41,6 +42,17 @@ def __init__(self, client, data, query=None): def arn(self): return self._arn_query.search(self.data) + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + asg_name = arn.split(':')[7].split('/')[1] + addon = dict(ResourceId=asg_name, + ResourceType='auto-scaling-group', + PropagateAtLaunch=False) + tags_list = [dict(Key=k, Value=str(v), **addon) for k, v in tags.items()] + return client.call('create_or_update_tags', Tags=tags_list) + class LaunchConfiguration(AWSResource): diff --git a/skew/resources/aws/cloudfront.py b/skew/resources/aws/cloudfront.py index 5b096fa..113efdb 100644 --- a/skew/resources/aws/cloudfront.py +++ b/skew/resources/aws/cloudfront.py @@ -1,6 +1,7 @@ import logging from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient LOG = logging.getLogger(__name__) @@ -34,3 +35,10 @@ class Meta(object): def filter(cls, arn, resource_id, data): LOG.debug('%s == %s', resource_id, data) return resource_id == data['Id'] + + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + tags_list = [dict(Key=k, Value=str(v)) for k, v in tags.items()] + return client.call('tag_resource', Resource=arn, Tags=dict(Items=tags_list)) diff --git a/skew/resources/resource.py b/skew/resources/resource.py index 92514cd..d1a4af0 100644 --- a/skew/resources/resource.py +++ b/skew/resources/resource.py @@ -69,6 +69,10 @@ def enumerate(cls, arn, region, account, resource_id=None, **kwargs): resources.append(cls(client, d, arn.query)) return resources + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + pass + class Meta(object): type = 'resource' dimension = None From 1b893fbeccb796d869cda79458c93536d2f068db Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Wed, 3 Oct 2018 14:06:17 +0200 Subject: [PATCH 06/62] Set the Name from the instance ID, because PrivateDnsName count be undefined, and PrivateIpAddress could be not unique --- skew/resources/aws/ec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skew/resources/aws/ec2.py b/skew/resources/aws/ec2.py index 5b520d9..747edfd 100644 --- a/skew/resources/aws/ec2.py +++ b/skew/resources/aws/ec2.py @@ -25,7 +25,7 @@ class Meta(object): id = 'InstanceId' filter_name = 'InstanceIds' filter_type = 'list' - name = 'PublicDnsName' + name = 'InstanceId' date = 'LaunchTime' dimension = 'InstanceId' From fb89799ca744efcd423e372ff3a644070f5ff299 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Wed, 3 Oct 2018 17:26:08 +0200 Subject: [PATCH 07/62] Define asset name from short Id/Name instead of fqdn --- skew/resources/aws/cloudfront.py | 2 +- skew/resources/aws/elb.py | 2 +- skew/resources/aws/elbv2.py | 2 +- skew/resources/aws/sqs.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/skew/resources/aws/cloudfront.py b/skew/resources/aws/cloudfront.py index 113efdb..865f950 100644 --- a/skew/resources/aws/cloudfront.py +++ b/skew/resources/aws/cloudfront.py @@ -26,7 +26,7 @@ class Meta(object): id = 'Id' tags_spec = ('list_tags_for_resource', 'Tags.Items[]', 'Resource', 'arn') - name = 'DomainName' + name = 'Id' filter_name = None date = 'LastModifiedTime' dimension = None diff --git a/skew/resources/aws/elb.py b/skew/resources/aws/elb.py index 527fc22..5381c4f 100644 --- a/skew/resources/aws/elb.py +++ b/skew/resources/aws/elb.py @@ -26,7 +26,7 @@ class Meta(object): id = 'LoadBalancerName' filter_name = 'LoadBalancerNames' filter_type = 'list' - name = 'DNSName' + name = 'LoadBalancerName' date = 'CreatedTime' dimension = 'LoadBalancerName' tags_spec = ('describe_tags', 'TagDescriptions[].Tags[]', diff --git a/skew/resources/aws/elbv2.py b/skew/resources/aws/elbv2.py index efebe24..72382ff 100644 --- a/skew/resources/aws/elbv2.py +++ b/skew/resources/aws/elbv2.py @@ -29,7 +29,7 @@ class Meta(object): id = 'LoadBalancerArn' filter_name = 'Names' filter_type = 'list' - name = 'DNSName' + name = 'LoadBalancerName' date = 'CreatedTime' dimension = None tags_spec = ('describe_tags', 'TagDescriptions[].Tags[]', diff --git a/skew/resources/aws/sqs.py b/skew/resources/aws/sqs.py index 9517bcb..4f22e07 100644 --- a/skew/resources/aws/sqs.py +++ b/skew/resources/aws/sqs.py @@ -25,7 +25,7 @@ class Meta(object): id = 'QueueUrl' filter_name = 'QueueNamePrefix' filter_type = 'scalar' - name = 'QueueUrl' + name = 'QueueName' date = None dimension = 'QueueName' From 97b5c2542dac6f18f542ca3067eaf644e224b34c Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 15 Oct 2018 12:07:37 +0200 Subject: [PATCH 08/62] Prefer short name --- skew/resources/aws/rds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skew/resources/aws/rds.py b/skew/resources/aws/rds.py index f6e856d..8683e39 100644 --- a/skew/resources/aws/rds.py +++ b/skew/resources/aws/rds.py @@ -27,7 +27,7 @@ class Meta(object): id = 'DBInstanceIdentifier' filter_name = 'DbInstanceIdentifier' filter_type = 'scalar' - name = 'Endpoint.Address' + name = 'DBInstanceIdentifier' date = 'InstanceCreateTime' dimension = 'DBInstanceIdentifier' From 48831a5c1699ba1c2f66a86c4c65f2d6474d0dbc Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 15 Oct 2018 12:08:02 +0200 Subject: [PATCH 09/62] Fix tags read --- skew/resources/aws/elbv2.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/skew/resources/aws/elbv2.py b/skew/resources/aws/elbv2.py index 72382ff..c9ebc67 100644 --- a/skew/resources/aws/elbv2.py +++ b/skew/resources/aws/elbv2.py @@ -33,7 +33,7 @@ class Meta(object): date = 'CreatedTime' dimension = None tags_spec = ('describe_tags', 'TagDescriptions[].Tags[]', - 'LoadBalancerNames', 'id') + 'ResourceArns', 'id') @property def arn(self): @@ -41,7 +41,6 @@ def arn(self): def __init__(self, client, data, query=None): super(LoadBalancer, self).__init__(client, data, query) - self._id = data detail_op, param_name, detail_path = self.Meta.detail_spec params = {param_name: self.data['LoadBalancerArn']} data = client.call(detail_op, **params) From f67c10845ca316ff489fd3feeef57a2c981b6f90 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 15 Oct 2018 12:12:54 +0200 Subject: [PATCH 10/62] Class method for tag writing --- skew/resources/aws/__init__.py | 1 + skew/resources/resource.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/skew/resources/aws/__init__.py b/skew/resources/aws/__init__.py index d3dd49c..496b839 100644 --- a/skew/resources/aws/__init__.py +++ b/skew/resources/aws/__init__.py @@ -47,6 +47,7 @@ class AWSResource(Resource): Each entry in the dictionary we define: * service - The AWS service in which this resource is defined. + * tags_support - Precise if the resource supports tags. * enum_spec - The enumeration configuration. This is a tuple consisting of the name of the operation to call to enumerate the resources, a jmespath query that will be run against the result of the operation diff --git a/skew/resources/resource.py b/skew/resources/resource.py index d1a4af0..b3ac6f8 100644 --- a/skew/resources/resource.py +++ b/skew/resources/resource.py @@ -71,7 +71,10 @@ def enumerate(cls, arn, region, account, resource_id=None, **kwargs): @classmethod def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): - pass + if hasattr(cls.Meta, 'tags_support') and (cls.Meta.tags_support): + client = skew.awsclient.get_awsclient( + 'resourcegroupstaggingapi', region, account, **kwargs) + client.call('tag_resources', ResourceARNList=[arn], Tags=tags) class Meta(object): type = 'resource' From 83ca0128a534cf40959a7f8b0361b3531d3d1b64 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 15 Oct 2018 17:50:03 +0200 Subject: [PATCH 11/62] Fix the ARN with specific convention --- skew/resources/aws/sqs.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/skew/resources/aws/sqs.py b/skew/resources/aws/sqs.py index 4f22e07..c4b4c90 100644 --- a/skew/resources/aws/sqs.py +++ b/skew/resources/aws/sqs.py @@ -34,3 +34,9 @@ def __init__(self, client, data, query=None): self.data = {self.Meta.id: data, 'QueueName': data.split('/')[-1]} self._id = self.data['QueueName'] + + @property + def arn(self): + return 'arn:aws:sqs:%s:%s:%s' % ( + self._client.region_name, + self._client.account_id, self.id) \ No newline at end of file From b7ee3641c6829d45fb4f67bd863383f9a3d24c3a Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 15 Oct 2018 17:52:01 +0200 Subject: [PATCH 12/62] Rename tags_support to resourcegroups_tagging --- skew/resources/aws/__init__.py | 2 +- skew/resources/resource.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/skew/resources/aws/__init__.py b/skew/resources/aws/__init__.py index 496b839..28adb8a 100644 --- a/skew/resources/aws/__init__.py +++ b/skew/resources/aws/__init__.py @@ -47,7 +47,7 @@ class AWSResource(Resource): Each entry in the dictionary we define: * service - The AWS service in which this resource is defined. - * tags_support - Precise if the resource supports tags. + * resourcegroups_tagging - Precise if the resource supports tags. * enum_spec - The enumeration configuration. This is a tuple consisting of the name of the operation to call to enumerate the resources, a jmespath query that will be run against the result of the operation diff --git a/skew/resources/resource.py b/skew/resources/resource.py index b3ac6f8..bb12171 100644 --- a/skew/resources/resource.py +++ b/skew/resources/resource.py @@ -71,10 +71,11 @@ def enumerate(cls, arn, region, account, resource_id=None, **kwargs): @classmethod def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): - if hasattr(cls.Meta, 'tags_support') and (cls.Meta.tags_support): + if hasattr(cls.Meta, 'resourcegroups_tagging') and (cls.Meta.resourcegroups_tagging): client = skew.awsclient.get_awsclient( 'resourcegroupstaggingapi', region, account, **kwargs) - client.call('tag_resources', ResourceARNList=[arn], Tags=tags) + r = client.call('tag_resources', ResourceARNList=[arn], Tags=tags) + logging.warn('Tag ARN %s : %s', arn, r) class Meta(object): type = 'resource' From 489a2b3923d02e02ed71b90e017e03d70d693ad8 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 15 Oct 2018 17:52:17 +0200 Subject: [PATCH 13/62] Enable Resource Groups Tagging (write) when possible --- skew/resources/aws/acm.py | 8 +------- skew/resources/aws/apigateway.py | 1 + skew/resources/aws/autoscaling.py | 1 + skew/resources/aws/cloudformation.py | 1 + skew/resources/aws/cloudfront.py | 1 + skew/resources/aws/cloudwatch.py | 1 + skew/resources/aws/dynamodb.py | 1 + skew/resources/aws/ec2.py | 14 ++++++++++++++ skew/resources/aws/ecs.py | 2 ++ skew/resources/aws/efs.py | 1 + skew/resources/aws/elasticache.py | 3 +++ skew/resources/aws/elasticbeanstalk.py | 8 +++++++- skew/resources/aws/elb.py | 1 + skew/resources/aws/elbv2.py | 2 ++ skew/resources/aws/es.py | 1 + skew/resources/aws/firehose.py | 1 + skew/resources/aws/iam.py | 6 ++++++ skew/resources/aws/kinesis.py | 1 + skew/resources/aws/lambda.py | 1 + skew/resources/aws/rds.py | 2 ++ skew/resources/aws/redshift.py | 1 + skew/resources/aws/route53.py | 3 +++ skew/resources/aws/s3.py | 1 + skew/resources/aws/ses.py | 1 + skew/resources/aws/sns.py | 2 ++ skew/resources/aws/sqs.py | 1 + skew/resources/aws/support.py | 1 + 27 files changed, 59 insertions(+), 8 deletions(-) diff --git a/skew/resources/aws/acm.py b/skew/resources/aws/acm.py index e754739..8402651 100644 --- a/skew/resources/aws/acm.py +++ b/skew/resources/aws/acm.py @@ -28,6 +28,7 @@ class Certificate(AWSResource): class Meta(object): service = 'acm' type = 'certificate' + resourcegroups_tagging = True enum_spec = ('list_certificates', 'CertificateSummaryList', None) detail_spec = None id = 'CertificateArn' @@ -41,10 +42,3 @@ class Meta(object): @property def arn(self): return self.data['CertificateArn'] - - @classmethod - def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): - client = get_awsclient( - cls.Meta.service, region, account, **kwargs) - tags_list = [dict(Key=k, Value=str(v)) for k, v in tags.items()] - return client.call('add_tags_to_certificate', CertificateArn=arn, Tags=tags_list) diff --git a/skew/resources/aws/apigateway.py b/skew/resources/aws/apigateway.py index a460a0a..0601868 100644 --- a/skew/resources/aws/apigateway.py +++ b/skew/resources/aws/apigateway.py @@ -21,6 +21,7 @@ class RestAPI(AWSResource): class Meta(object): service = 'apigateway' type = 'restapis' + resourcegroups_tagging = False enum_spec = ('get_rest_apis', 'items', None) id = 'id' filter_name = None diff --git a/skew/resources/aws/autoscaling.py b/skew/resources/aws/autoscaling.py index 54a191a..31d4385 100644 --- a/skew/resources/aws/autoscaling.py +++ b/skew/resources/aws/autoscaling.py @@ -23,6 +23,7 @@ class AutoScalingGroup(AWSResource): class Meta(object): service = 'autoscaling' type = 'autoScalingGroup' + resourcegroups_tagging = False name = 'AutoScalingGroupName' date = 'CreatedTime' dimension = 'AutoScalingGroupName' diff --git a/skew/resources/aws/cloudformation.py b/skew/resources/aws/cloudformation.py index 782af9f..96bc1e5 100644 --- a/skew/resources/aws/cloudformation.py +++ b/skew/resources/aws/cloudformation.py @@ -38,6 +38,7 @@ def enumerate(cls, arn, region, account, resource_id=None, **kwargs): class Meta(object): service = 'cloudformation' type = 'stack' + resourcegroups_tagging = False enum_spec = ('describe_stacks', 'Stacks[]', None) detail_spec = ('describe_stack_resources', 'StackName', 'StackResources[]') diff --git a/skew/resources/aws/cloudfront.py b/skew/resources/aws/cloudfront.py index 865f950..61f4793 100644 --- a/skew/resources/aws/cloudfront.py +++ b/skew/resources/aws/cloudfront.py @@ -21,6 +21,7 @@ class Distribution(CloudfrontResource): class Meta(object): service = 'cloudfront' type = 'distribution' + resourcegroups_tagging = False enum_spec = ('list_distributions', 'DistributionList.Items[]', None) detail_spec = None id = 'Id' diff --git a/skew/resources/aws/cloudwatch.py b/skew/resources/aws/cloudwatch.py index 89807bb..69916d0 100644 --- a/skew/resources/aws/cloudwatch.py +++ b/skew/resources/aws/cloudwatch.py @@ -20,6 +20,7 @@ class Alarm(AWSResource): class Meta(object): service = 'cloudwatch' type = 'alarm' + resourcegroups_tagging = False enum_spec = ('describe_alarms', 'MetricAlarms', None) id = 'AlarmArn' filter_name = 'AlarmNames' diff --git a/skew/resources/aws/dynamodb.py b/skew/resources/aws/dynamodb.py index 196c6b7..074ed69 100644 --- a/skew/resources/aws/dynamodb.py +++ b/skew/resources/aws/dynamodb.py @@ -27,6 +27,7 @@ class Table(AWSResource): class Meta(object): service = 'dynamodb' type = 'table' + resourcegroups_tagging = True enum_spec = ('list_tables', 'TableNames', None) detail_spec = ('describe_table', 'TableName', 'Table') id = 'Table' diff --git a/skew/resources/aws/ec2.py b/skew/resources/aws/ec2.py index 747edfd..3bccb1e 100644 --- a/skew/resources/aws/ec2.py +++ b/skew/resources/aws/ec2.py @@ -20,6 +20,7 @@ class Instance(AWSResource): class Meta(object): service = 'ec2' type = 'instance' + resourcegroups_tagging = True enum_spec = ('describe_instances', 'Reservations[].Instances[]', None) detail_spec = None id = 'InstanceId' @@ -39,6 +40,7 @@ class SecurityGroup(AWSResource): class Meta(object): service = 'ec2' type = 'security-group' + resourcegroups_tagging = True enum_spec = ('describe_security_groups', 'SecurityGroups', None) detail_spec = None id = 'GroupId' @@ -54,6 +56,7 @@ class KeyPair(AWSResource): class Meta(object): service = 'ec2' type = 'key-pair' + resourcegroups_tagging = False enum_spec = ('describe_key_pairs', 'KeyPairs', None) detail_spec = None id = 'KeyName' @@ -68,6 +71,7 @@ class Address(AWSResource): class Meta(object): service = 'ec2' type = 'address' + resourcegroups_tagging = False enum_spec = ('describe_addresses', 'Addresses', None) detail_spec = None id = 'PublicIp' @@ -83,6 +87,7 @@ class Volume(AWSResource): class Meta(object): service = 'ec2' type = 'volume' + resourcegroups_tagging = True enum_spec = ('describe_volumes', 'Volumes', None) detail_spec = None id = 'VolumeId' @@ -105,6 +110,7 @@ class Snapshot(AWSResource): class Meta(object): service = 'ec2' type = 'snapshot' + resourcegroups_tagging = True enum_spec = ( 'describe_snapshots', 'Snapshots', {'OwnerIds': ['self']}) detail_spec = None @@ -128,6 +134,7 @@ class Image(AWSResource): class Meta(object): service = 'ec2' type = 'image' + resourcegroups_tagging = True enum_spec = ( 'describe_images', 'Images', {'Owners': ['self']}) detail_spec = None @@ -151,6 +158,7 @@ class Vpc(AWSResource): class Meta(object): service = 'ec2' type = 'vpc' + resourcegroups_tagging = True enum_spec = ('describe_vpcs', 'Vpcs', None) detail_spec = None id = 'VpcId' @@ -166,6 +174,7 @@ class Subnet(AWSResource): class Meta(object): service = 'ec2' type = 'subnet' + resourcegroups_tagging = True enum_spec = ('describe_subnets', 'Subnets', None) detail_spec = None id = 'SubnetId' @@ -181,6 +190,7 @@ class CustomerGateway(AWSResource): class Meta(object): service = 'ec2' type = 'customer-gateway' + resourcegroups_tagging = True enum_spec = ('describe_customer_gateways', 'CustomerGateway', None) detail_spec = None id = 'CustomerGatewayId' @@ -196,6 +206,7 @@ class InternetGateway(AWSResource): class Meta(object): service = 'ec2' type = 'internet-gateway' + resourcegroups_tagging = True enum_spec = ('describe_internet_gateways', 'InternetGateway', None) detail_spec = None id = 'InternetGatewayId' @@ -211,6 +222,7 @@ class RouteTable(AWSResource): class Meta(object): service = 'ec2' type = 'route-table' + resourcegroups_tagging = True enum_spec = ('describe_route_tables', 'RouteTables', None) detail_spec = None id = 'RouteTableId' @@ -226,6 +238,7 @@ class NetworkAcl(AWSResource): class Meta(object): service = 'ec2' type = 'network-acl' + resourcegroups_tagging = True enum_spec = ('describe_network_acls', 'NetworkAcls', None) detail_spec = None id = 'NetworkAclId' @@ -241,6 +254,7 @@ class VpcPeeringConnection(AWSResource): class Meta(object): service = 'ec2' type = 'vpc-peering-connection' + resourcegroups_tagging = False enum_spec = ('describe_vpc_peering_connections', 'VpcPeeringConnection', None) detail_spec = None diff --git a/skew/resources/aws/ecs.py b/skew/resources/aws/ecs.py index 91663dd..7927755 100644 --- a/skew/resources/aws/ecs.py +++ b/skew/resources/aws/ecs.py @@ -27,6 +27,7 @@ class Cluster(AWSResource): class Meta(object): service = 'ecs' type = 'cluster' + resourcegroups_tagging = False enum_spec = ('list_clusters', 'clusterArns', None) detail_spec = ('describe_clusters', 'clusters', 'clusters[0]') id = None @@ -55,6 +56,7 @@ class TaskDefinition(AWSResource): class Meta(object): service = 'ecs' type = 'task-definition' + resourcegroups_tagging = False enum_spec = ('list_task_definitions', 'taskDefinitionArns', None) detail_spec = ('describe_task_definition', 'taskDefinition', 'taskDefinition') id = None diff --git a/skew/resources/aws/efs.py b/skew/resources/aws/efs.py index e5c21bd..38d87ba 100644 --- a/skew/resources/aws/efs.py +++ b/skew/resources/aws/efs.py @@ -27,6 +27,7 @@ class Filesystem(AWSResource): class Meta(object): service = 'efs' type = 'filesystem' + resourcegroups_tagging = False enum_spec = ('describe_file_systems', 'FileSystems', None) detail_spec = None id = 'FileSystemId' diff --git a/skew/resources/aws/elasticache.py b/skew/resources/aws/elasticache.py index bbfbfc9..7f5141f 100644 --- a/skew/resources/aws/elasticache.py +++ b/skew/resources/aws/elasticache.py @@ -20,6 +20,7 @@ class Cluster(AWSResource): class Meta(object): service = 'elasticache' type = 'cluster' + resourcegroups_tagging = True enum_spec = ('describe_cache_clusters', 'CacheClusters[]', None) detail_spec = None @@ -45,6 +46,7 @@ class SubnetGroup(AWSResource): class Meta(object): service = 'elasticache' type = 'subnet-group' + resourcegroups_tagging = False enum_spec = ('describe_cache_subnet_groups', 'CacheSubnetGroups', None) detail_spec = None @@ -61,6 +63,7 @@ class Snapshot(AWSResource): class Meta(object): service = 'elasticache' type = 'snapshot' + resourcegroups_tagging = True enum_spec = ('describe_snapshots', 'Snapshots', None) detail_spec = None id = 'SnapshotName' diff --git a/skew/resources/aws/elasticbeanstalk.py b/skew/resources/aws/elasticbeanstalk.py index be2e598..7b8c31a 100644 --- a/skew/resources/aws/elasticbeanstalk.py +++ b/skew/resources/aws/elasticbeanstalk.py @@ -17,6 +17,7 @@ class Application(AWSResource): class Meta(object): service = 'elasticbeanstalk' type = 'application' + resourcegroups_tagging = False enum_spec = ('describe_applications', 'Applications', None) detail_spec = None id = 'ApplicationName' @@ -31,6 +32,7 @@ class Environment(AWSResource): class Meta(object): service = 'elasticbeanstalk' type = 'environment' + resourcegroups_tagging = True enum_spec = ('describe_environments', 'Environments', None) detail_spec = None id = 'EnvironmentName' @@ -38,4 +40,8 @@ class Meta(object): filter_type = None name = 'EnvironmentName' date = None - dimension = None \ No newline at end of file + dimension = None + + @property + def arn(self): + return self.data['EnvironmentArn'] diff --git a/skew/resources/aws/elb.py b/skew/resources/aws/elb.py index 5381c4f..c03db69 100644 --- a/skew/resources/aws/elb.py +++ b/skew/resources/aws/elb.py @@ -20,6 +20,7 @@ class LoadBalancer(AWSResource): class Meta(object): service = 'elb' type = 'loadbalancer' + resourcegroups_tagging = True enum_spec = ('describe_load_balancers', 'LoadBalancerDescriptions', None) detail_spec = None diff --git a/skew/resources/aws/elbv2.py b/skew/resources/aws/elbv2.py index c9ebc67..bb2d4de 100644 --- a/skew/resources/aws/elbv2.py +++ b/skew/resources/aws/elbv2.py @@ -22,6 +22,7 @@ class LoadBalancer(AWSResource): class Meta(object): service = 'elbv2' type = 'loadbalancer' + resourcegroups_tagging = True enum_spec = ('describe_load_balancers', 'LoadBalancers', None) # detail_spec = None @@ -52,6 +53,7 @@ class TargetGroup(AWSResource): class Meta(object): service = 'elbv2' type = 'targetgroup' + resourcegroups_tagging = False enum_spec = ('describe_target_groups', 'TargetGroups', None) detail_spec = None diff --git a/skew/resources/aws/es.py b/skew/resources/aws/es.py index 3243ebf..8a13247 100644 --- a/skew/resources/aws/es.py +++ b/skew/resources/aws/es.py @@ -22,6 +22,7 @@ class ElasticsearchDomain(AWSResource): class Meta(object): service = 'es' type = 'domain' + resourcegroups_tagging = True enum_spec = ('list_domain_names', 'DomainNames[].DomainName', None) tags_spec = ('list_tags', 'TagList', 'ARN', 'arn') diff --git a/skew/resources/aws/firehose.py b/skew/resources/aws/firehose.py index f498b9d..103f7dc 100644 --- a/skew/resources/aws/firehose.py +++ b/skew/resources/aws/firehose.py @@ -18,6 +18,7 @@ class DeliveryStream(AWSResource): class Meta(object): service = 'firehose' type = 'deliverystream' + resourcegroups_tagging = False enum_spec = ('list_delivery_streams', 'DeliveryStreamNames', None) detail_spec = ('describe_delivery_stream', 'DeliveryStreamName', 'DeliveryStreamDescription') id = 'DeliveryStreamName' diff --git a/skew/resources/aws/iam.py b/skew/resources/aws/iam.py index 3485640..cd70514 100644 --- a/skew/resources/aws/iam.py +++ b/skew/resources/aws/iam.py @@ -34,6 +34,7 @@ class Group(IAMResource): class Meta(object): service = 'iam' type = 'group' + resourcegroups_tagging = False enum_spec = ('list_groups', 'Groups', None) detail_spec = None id = 'GroupId' @@ -53,6 +54,7 @@ class User(IAMResource): class Meta(object): service = 'iam' type = 'user' + resourcegroups_tagging = False enum_spec = ('list_users', 'Users', None) detail_spec = None id = 'UserId' @@ -72,6 +74,7 @@ class Role(IAMResource): class Meta(object): service = 'iam' type = 'role' + resourcegroups_tagging = False enum_spec = ('list_roles', 'Roles', None) detail_spec = None id = 'RoleId' @@ -91,6 +94,7 @@ class InstanceProfile(IAMResource): class Meta(object): service = 'iam' type = 'instance-profile' + resourcegroups_tagging = False enum_spec = ('list_instance_profiles', 'InstanceProfiles', None) detail_spec = None id = 'InstanceProfileId' @@ -110,6 +114,7 @@ class Policy(IAMResource): class Meta(object): service = 'iam' type = 'policy' + resourcegroups_tagging = False enum_spec = ('list_policies', 'Policies', None) detail_spec = None id = 'PolicyId' @@ -129,6 +134,7 @@ class ServerCertificate(IAMResource): class Meta(object): service = 'iam' type = 'server-certificate' + resourcegroups_tagging = False enum_spec = ('list_server_certificates', 'ServerCertificateMetadataList', None) diff --git a/skew/resources/aws/kinesis.py b/skew/resources/aws/kinesis.py index 69f009c..4d6f4b3 100644 --- a/skew/resources/aws/kinesis.py +++ b/skew/resources/aws/kinesis.py @@ -20,6 +20,7 @@ class Stream(AWSResource): class Meta(object): service = 'kinesis' type = 'stream' + resourcegroups_tagging = True enum_spec = ('list_streams', 'StreamNames', None) detail_spec = None id = 'StreamName' diff --git a/skew/resources/aws/lambda.py b/skew/resources/aws/lambda.py index c42cc87..b3cfe4d 100644 --- a/skew/resources/aws/lambda.py +++ b/skew/resources/aws/lambda.py @@ -37,6 +37,7 @@ def enumerate(cls, arn, region, account, resource_id=None, **kwargs): class Meta(object): service = 'lambda' type = 'function' + resourcegroups_tagging = True enum_spec = ('list_functions', 'Functions', None) detail_spec = None id = 'FunctionName' diff --git a/skew/resources/aws/rds.py b/skew/resources/aws/rds.py index 8683e39..8df6ff4 100644 --- a/skew/resources/aws/rds.py +++ b/skew/resources/aws/rds.py @@ -20,6 +20,7 @@ class DBInstance(AWSResource): class Meta(object): service = 'rds' type = 'db' + resourcegroups_tagging = True enum_spec = ('describe_db_instances', 'DBInstances', None) tags_spec = ('list_tags_for_resource', 'TagList', 'ResourceName', 'arn') @@ -44,6 +45,7 @@ class DBSecurityGroup(AWSResource): class Meta(object): service = 'rds' type = 'secgrp' + resourcegroups_tagging = True enum_spec = ('describe_db_security_groups', 'DBSecurityGroups', None) detail_spec = None id = 'DBSecurityGroupName' diff --git a/skew/resources/aws/redshift.py b/skew/resources/aws/redshift.py index a39430e..07e974d 100644 --- a/skew/resources/aws/redshift.py +++ b/skew/resources/aws/redshift.py @@ -20,6 +20,7 @@ class Cluster(AWSResource): class Meta(object): service = 'redshift' type = 'cluster' + resourcegroups_tagging = True enum_spec = ('describe_clusters', 'Clusters', None) detail_spec = None id = 'ClusterIdentifier' diff --git a/skew/resources/aws/route53.py b/skew/resources/aws/route53.py index 2874ef3..21460fe 100644 --- a/skew/resources/aws/route53.py +++ b/skew/resources/aws/route53.py @@ -28,6 +28,7 @@ class HostedZone(Route53Resource): class Meta(object): service = 'route53' type = 'hostedzone' + resourcegroups_tagging = True enum_spec = ('list_hosted_zones', 'HostedZones', None) detail_spec = ('GetHostedZone', 'Id', None) id = 'Id' @@ -52,6 +53,7 @@ class HealthCheck(Route53Resource): class Meta(object): service = 'route53' type = 'healthcheck' + resourcegroups_tagging = True enum_spec = ('list_health_checks', 'HealthChecks', None) detail_spec = ('GetHealthCheck', 'Id', None) id = 'Id' @@ -68,6 +70,7 @@ class ResourceRecordSet(Route53Resource): class Meta(object): service = 'route53' type = 'rrset' + resourcegroups_tagging = False enum_spec = ('list_resource_record_sets', 'ResourceRecordSets', None) detail_spec = None id = 'Name' diff --git a/skew/resources/aws/s3.py b/skew/resources/aws/s3.py index 3057752..0b22922 100644 --- a/skew/resources/aws/s3.py +++ b/skew/resources/aws/s3.py @@ -49,6 +49,7 @@ def enumerate(cls, arn, region, account, resource_id=None, **kwargs): class Meta(object): service = 's3' type = 'bucket' + resourcegroups_tagging = True enum_spec = ('list_buckets', 'Buckets[]', None) detail_spec = ('list_objects', 'Bucket', 'Contents[]') id = 'Name' diff --git a/skew/resources/aws/ses.py b/skew/resources/aws/ses.py index 90a575a..ec7608c 100644 --- a/skew/resources/aws/ses.py +++ b/skew/resources/aws/ses.py @@ -27,6 +27,7 @@ class Identity(AWSResource): class Meta(object): service = 'ses' type = 'identity' + resourcegroups_tagging = False enum_spec = ('list_identities', 'Identities', None) detail_spec = ('describe_table', 'TableName', 'Table') id = 'Identity' diff --git a/skew/resources/aws/sns.py b/skew/resources/aws/sns.py index cf0c8ef..216e6d6 100644 --- a/skew/resources/aws/sns.py +++ b/skew/resources/aws/sns.py @@ -23,6 +23,7 @@ class Topic(AWSResource): class Meta(object): service = 'sns' type = 'topic' + resourcegroups_tagging = False enum_spec = ('list_topics', 'Topics', None) detail_spec = ('get_topic_attributes', 'TopicArn', 'Attributes') id = 'TopicArn' @@ -61,6 +62,7 @@ class Subscription(AWSResource): class Meta(object): service = 'sns' type = 'subscription' + resourcegroups_tagging = False enum_spec = ('list_subscriptions', 'Subscriptions', None) detail_spec = ('get_subscription_attributes', 'SubscriptionArn', 'Attributes') diff --git a/skew/resources/aws/sqs.py b/skew/resources/aws/sqs.py index c4b4c90..4f6e5af 100644 --- a/skew/resources/aws/sqs.py +++ b/skew/resources/aws/sqs.py @@ -20,6 +20,7 @@ class Queue(AWSResource): class Meta(object): service = 'sqs' type = 'queue' + resourcegroups_tagging = True enum_spec = ('list_queues', 'QueueUrls', None) detail_spec = ('get_queue_attributes', 'QueueUrl', 'QueueUrl') id = 'QueueUrl' diff --git a/skew/resources/aws/support.py b/skew/resources/aws/support.py index 61733e6..d957140 100644 --- a/skew/resources/aws/support.py +++ b/skew/resources/aws/support.py @@ -27,6 +27,7 @@ class Check(AWSResource): class Meta(object): service = 'support' type = 'check' + resourcegroups_tagging = False enum_spec = ('describe_trusted_advisor_checks', 'checks', None) detail_spec = None id = 'id' From 29de51d72be01b605f668df566725a634492e810 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 22 Oct 2018 17:05:38 +0200 Subject: [PATCH 14/62] Sleek datas frequently varied --- skew/resources/aws/efs.py | 4 ++++ skew/resources/aws/rds.py | 2 ++ skew/resources/resource.py | 6 ++++++ 3 files changed, 12 insertions(+) diff --git a/skew/resources/aws/efs.py b/skew/resources/aws/efs.py index 38d87ba..f2c9d72 100644 --- a/skew/resources/aws/efs.py +++ b/skew/resources/aws/efs.py @@ -17,6 +17,7 @@ import jmespath from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient LOG = logging.getLogger(__name__) @@ -46,3 +47,6 @@ def arn(self): self._client.region_name, self._client.account_id, 'file-system-id', self.id) + + def sleek(self): + self.data['SizeInBytes'] = 0 diff --git a/skew/resources/aws/rds.py b/skew/resources/aws/rds.py index 8df6ff4..425955f 100644 --- a/skew/resources/aws/rds.py +++ b/skew/resources/aws/rds.py @@ -62,3 +62,5 @@ def arn(self): self._client.region_name, self._client.account_id, self.resourcetype, self.id) + def sleek(self): + self.data['LatestRestorableTime'] = '' diff --git a/skew/resources/resource.py b/skew/resources/resource.py index bb12171..b3b9c3c 100644 --- a/skew/resources/resource.py +++ b/skew/resources/resource.py @@ -77,6 +77,12 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): r = client.call('tag_resources', ResourceARNList=[arn], Tags=tags) logging.warn('Tag ARN %s : %s', arn, r) + def sleek(self): + """ + overrides datas frequently varied by a static value. + """ + pass + class Meta(object): type = 'resource' dimension = None From 297f4042042bcf9df7e7287ba371e06b948a95c0 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 22 Oct 2018 17:07:13 +0200 Subject: [PATCH 15/62] Add set tag feature for EFS --- skew/resources/aws/efs.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/skew/resources/aws/efs.py b/skew/resources/aws/efs.py index f2c9d72..58e9ac0 100644 --- a/skew/resources/aws/efs.py +++ b/skew/resources/aws/efs.py @@ -50,3 +50,12 @@ def arn(self): def sleek(self): self.data['SizeInBytes'] = 0 + + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + tags_list = [dict(Key=k, Value=str(v)) for k, v in tags.items()] + x = client.call('create_tags', FileSystemId=arn.split('/')[-1], Tags=tags_list) + print(x, arn.split('/')[-1], tags_list) + return x From 72a7f837c4ba4a2e109411fea56634a0b36ee8e1 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Tue, 23 Oct 2018 10:46:38 +0200 Subject: [PATCH 16/62] Add OpsWorks stack --- skew/resources/__init__.py | 1 + skew/resources/aws/opsworks.py | 41 ++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 skew/resources/aws/opsworks.py diff --git a/skew/resources/__init__.py b/skew/resources/__init__.py index e145983..e6fbac1 100644 --- a/skew/resources/__init__.py +++ b/skew/resources/__init__.py @@ -59,6 +59,7 @@ 'aws.iam.server-certificate': 'aws.iam.ServerCertificate', 'aws.kinesis.stream': 'aws.kinesis.Stream', 'aws.lambda.function': 'aws.lambda.Function', + 'aws.opsworks.stack': 'aws.opsworks.Stack', 'aws.rds.db': 'aws.rds.DBInstance', 'aws.rds.secgrp': 'aws.rds.DBSecurityGroup', 'aws.redshift.cluster': 'aws.redshift.Cluster', diff --git a/skew/resources/aws/opsworks.py b/skew/resources/aws/opsworks.py new file mode 100644 index 0000000..60b4b79 --- /dev/null +++ b/skew/resources/aws/opsworks.py @@ -0,0 +1,41 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Stack(AWSResource): + + class Meta(object): + service = 'opsworks' + type = 'stack' + resourcegroups_tagging = True # only support regional, not classic (us-east-1) + enum_spec = ('describe_stacks', 'Stacks', None) + detail_spec = None + id = 'StackId' + filter_name = None + name = 'Name' + date = 'CreatedAt' + dimension = None + tags_spec = ('list_tags', 'Tags', + 'ResourceArn', 'arn') + + @property + def arn(self): + return self.data.get('Arn') From 0856ccc5d39ae180b91237f6aac732894655c782 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Tue, 23 Oct 2018 10:47:28 +0200 Subject: [PATCH 17/62] Fix the ARN with specific convention --- skew/resources/aws/apigateway.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/skew/resources/aws/apigateway.py b/skew/resources/aws/apigateway.py index 0601868..2a61d9d 100644 --- a/skew/resources/aws/apigateway.py +++ b/skew/resources/aws/apigateway.py @@ -36,3 +36,10 @@ def filter(cls, arn, resource_id, data): api_id = data.get(cls.Meta.id) LOG.debug('%s == %s', resource_id, api_id) return resource_id == api_id + + @property + def arn(self): + # arn:aws:apigateway:us-east-1::/restapis/vwxyz12345 + return 'arn:aws:apigateway:%s::/restapis/%s' % ( + self._client.region_name, + self.id) From 08296e918893bce7dd7a4d9b43476de2604934fc Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Tue, 23 Oct 2018 11:44:18 +0200 Subject: [PATCH 18/62] Adjust logging --- skew/resources/resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skew/resources/resource.py b/skew/resources/resource.py index b3b9c3c..d4af8c2 100644 --- a/skew/resources/resource.py +++ b/skew/resources/resource.py @@ -75,7 +75,7 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): client = skew.awsclient.get_awsclient( 'resourcegroupstaggingapi', region, account, **kwargs) r = client.call('tag_resources', ResourceARNList=[arn], Tags=tags) - logging.warn('Tag ARN %s : %s', arn, r) + LOG.debug('Tag ARN %s, r=%s', arn, r) def sleek(self): """ From 67901bbe0f6a59fc992d25e2ac374ccba1ad79c1 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Tue, 23 Oct 2018 11:44:30 +0200 Subject: [PATCH 19/62] Tags using OpsWorks API instead of ResourceGroupsTaggingAPI --- skew/resources/aws/opsworks.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/skew/resources/aws/opsworks.py b/skew/resources/aws/opsworks.py index 60b4b79..59cec74 100644 --- a/skew/resources/aws/opsworks.py +++ b/skew/resources/aws/opsworks.py @@ -15,6 +15,7 @@ import logging from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient LOG = logging.getLogger(__name__) @@ -25,7 +26,7 @@ class Stack(AWSResource): class Meta(object): service = 'opsworks' type = 'stack' - resourcegroups_tagging = True # only support regional, not classic (us-east-1) + resourcegroups_tagging = False enum_spec = ('describe_stacks', 'Stacks', None) detail_spec = None id = 'StackId' @@ -39,3 +40,12 @@ class Meta(object): @property def arn(self): return self.data.get('Arn') + + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + # ResourceGroupsTaggingAPI supports regional stacks, but not classic (us-east-1) + # opsworks.tag_resource() supports both + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + r = client.call('tag_resource', ResourceArn=arn, Tags=tags) + LOG.debug('Tag ARN %s, r=%s', arn, r) From 0e3feaa0f979ed7aaa94e55ed02474d8cc8e9045 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Tue, 23 Oct 2018 16:18:12 +0200 Subject: [PATCH 20/62] Fix bucket Name --- skew/resources/aws/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skew/resources/aws/s3.py b/skew/resources/aws/s3.py index 0b22922..e174419 100644 --- a/skew/resources/aws/s3.py +++ b/skew/resources/aws/s3.py @@ -54,7 +54,7 @@ class Meta(object): detail_spec = ('list_objects', 'Bucket', 'Contents[]') id = 'Name' filter_name = None - name = 'BucketName' + name = 'Name' date = 'CreationDate' dimension = None tags_spec = ('get_bucket_tagging', 'TagSet[]', From bcc0ad79c72d94a92c74c32729aa9129089c3b57 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Fri, 16 Nov 2018 10:40:29 +0100 Subject: [PATCH 21/62] Set EC2 Instance name from its tag Name (or InstanceId) --- skew/resources/aws/ec2.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/skew/resources/aws/ec2.py b/skew/resources/aws/ec2.py index 3bccb1e..ea6fe71 100644 --- a/skew/resources/aws/ec2.py +++ b/skew/resources/aws/ec2.py @@ -34,6 +34,10 @@ class Meta(object): def parent(self): return self.data['ImageId'] + def __init__(self, client, data, query=None): + super(Instance, self).__init__(client, data, query) + # Asset name is get by tags if defined, or is InstanceId + self._name = self.tags.get('Name', self.data['InstanceId']) class SecurityGroup(AWSResource): From ddfb4565ae2d963f1b361b9f3c6ea0b1478a6160 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Wed, 21 Nov 2018 10:59:26 +0100 Subject: [PATCH 22/62] Fix sleek, set at DBInstance instead of DBSecurityGroup --- skew/resources/aws/rds.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/skew/resources/aws/rds.py b/skew/resources/aws/rds.py index 425955f..3898a54 100644 --- a/skew/resources/aws/rds.py +++ b/skew/resources/aws/rds.py @@ -39,6 +39,9 @@ def arn(self): self._client.region_name, self._client.account_id, self.resourcetype, self.id) + def sleek(self): + self.data['LatestRestorableTime'] = '' + class DBSecurityGroup(AWSResource): @@ -61,6 +64,3 @@ def arn(self): self._client.service_name, self._client.region_name, self._client.account_id, self.resourcetype, self.id) - - def sleek(self): - self.data['LatestRestorableTime'] = '' From b503c7a58cf1104c9673ae504bb2bd2fd21c117a Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Thu, 29 Nov 2018 09:35:54 +0100 Subject: [PATCH 23/62] Implement in sleek method the lists consitant ordering --- skew/resources/aws/autoscaling.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/skew/resources/aws/autoscaling.py b/skew/resources/aws/autoscaling.py index 31d4385..1a1b8ca 100644 --- a/skew/resources/aws/autoscaling.py +++ b/skew/resources/aws/autoscaling.py @@ -34,8 +34,6 @@ class Meta(object): filter_type = 'list' def __init__(self, client, data, query=None): - # Always save the list in the same order to avoid false changes detection - data['EnabledMetrics'].sort(key=lambda item:item['Metric']) super(AutoScalingGroup, self).__init__(client, data, query) self._arn_query = jmespath.compile('AutoScalingGroupARN') @@ -43,6 +41,11 @@ def __init__(self, client, data, query=None): def arn(self): return self._arn_query.search(self.data) + def sleek(self): + # Always render lists in the same order to avoid false changes detection + self.data['EnabledMetrics'].sort(key=lambda item: item['Metric']) + self.data['SuspendedProcesses'].sort(key=str) + @classmethod def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): client = get_awsclient( From 8ba2041238f42d9f1cf23d486242feff213408c0 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Wed, 26 Dec 2018 09:18:48 +0100 Subject: [PATCH 24/62] Compatibility fix for ECS, that use lowercase 'key' and 'value' as dict keys --- skew/resources/aws/__init__.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/skew/resources/aws/__init__.py b/skew/resources/aws/__init__.py index 28adb8a..a2eadc2 100644 --- a/skew/resources/aws/__init__.py +++ b/skew/resources/aws/__init__.py @@ -170,12 +170,16 @@ def tags(self): _tags = self.data['Tags'] if isinstance(_tags, list): for kvpair in _tags: - if kvpair['Key'] in self._tags: - if not isinstance(self._tags[kvpair['Key']], list): - self._tags[kvpair['Key']] = [self._tags[kvpair['Key']]] - self._tags[kvpair['Key']].append(kvpair['Value']) + # Compatibility fix for ECS, that use lowercase 'key' and 'value' as dict keys + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ecs.html#ECS.Client.list_tags_for_resource + tags_key = kvpair.get('Key', kvpair.get('key')) + tags_value = kvpair.get('Value', kvpair.get('value')) + if tags_key in self._tags: + if not isinstance(self._tags[tags_key], list): + self._tags[tags_key] = [self._tags[tags_key]] + self._tags[tags_key].append(tags_value) else: - self._tags[kvpair['Key']] = kvpair['Value'] + self._tags[tags_key] = tags_value elif isinstance(_tags, dict): self._tags = _tags return self._tags From 94298a224eeaaa7908feb4ca9776095159faf5f4 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Wed, 26 Dec 2018 09:20:49 +0100 Subject: [PATCH 25/62] Add tagging support for ECS --- skew/resources/aws/ecs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/skew/resources/aws/ecs.py b/skew/resources/aws/ecs.py index 7927755..046137d 100644 --- a/skew/resources/aws/ecs.py +++ b/skew/resources/aws/ecs.py @@ -31,7 +31,8 @@ class Meta(object): enum_spec = ('list_clusters', 'clusterArns', None) detail_spec = ('describe_clusters', 'clusters', 'clusters[0]') id = None - + tags_spec = ('list_tags_for_resource', 'tags[]', + 'resourceArn', 'arn') filter_name = None name = 'clusterName' From 6c03255650a49162b4a03737e21bdaeb45ff2036 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Wed, 26 Dec 2018 09:21:14 +0100 Subject: [PATCH 26/62] New method unset_tags() --- skew/resources/aws/autoscaling.py | 11 +++++++++++ skew/resources/aws/cloudfront.py | 6 ++++++ skew/resources/aws/efs.py | 8 ++++++++ skew/resources/aws/opsworks.py | 9 +++++++++ skew/resources/resource.py | 8 ++++++++ 5 files changed, 42 insertions(+) diff --git a/skew/resources/aws/autoscaling.py b/skew/resources/aws/autoscaling.py index 1a1b8ca..0893525 100644 --- a/skew/resources/aws/autoscaling.py +++ b/skew/resources/aws/autoscaling.py @@ -57,6 +57,17 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): tags_list = [dict(Key=k, Value=str(v), **addon) for k, v in tags.items()] return client.call('create_or_update_tags', Tags=tags_list) + @classmethod + def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + asg_name = arn.split(':')[7].split('/')[1] + addon = dict(ResourceId=asg_name, + ResourceType='auto-scaling-group', + PropagateAtLaunch=False) + tags_list = [dict(Key=k, Value=str(''), **addon) for k in tags_keys] + return client.call('delete_tags', Tags=tags_list) + class LaunchConfiguration(AWSResource): diff --git a/skew/resources/aws/cloudfront.py b/skew/resources/aws/cloudfront.py index 61f4793..6e9b050 100644 --- a/skew/resources/aws/cloudfront.py +++ b/skew/resources/aws/cloudfront.py @@ -43,3 +43,9 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): cls.Meta.service, region, account, **kwargs) tags_list = [dict(Key=k, Value=str(v)) for k, v in tags.items()] return client.call('tag_resource', Resource=arn, Tags=dict(Items=tags_list)) + + @classmethod + def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + return client.call('untag_resource', Resource=arn, TagsKeys=dict(Items=tags_keys)) diff --git a/skew/resources/aws/efs.py b/skew/resources/aws/efs.py index 58e9ac0..f706132 100644 --- a/skew/resources/aws/efs.py +++ b/skew/resources/aws/efs.py @@ -59,3 +59,11 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): x = client.call('create_tags', FileSystemId=arn.split('/')[-1], Tags=tags_list) print(x, arn.split('/')[-1], tags_list) return x + + @classmethod + def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + x = client.call('delete_tags', FileSystemId=arn.split('/')[-1], TagsKeys=tags_keys) + print(x, arn.split('/')[-1], tags_keys) + return x diff --git a/skew/resources/aws/opsworks.py b/skew/resources/aws/opsworks.py index 59cec74..08441ae 100644 --- a/skew/resources/aws/opsworks.py +++ b/skew/resources/aws/opsworks.py @@ -49,3 +49,12 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): cls.Meta.service, region, account, **kwargs) r = client.call('tag_resource', ResourceArn=arn, Tags=tags) LOG.debug('Tag ARN %s, r=%s', arn, r) + + @classmethod + def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): + # ResourceGroupsTaggingAPI supports regional stacks, but not classic (us-east-1) + # opsworks.untag_resource() supports both + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + r = client.call('untag_resource', ResourceArn=arn, TagsKeys=tags_keys) + LOG.debug('UnTag ARN %s, r=%s', arn, r) diff --git a/skew/resources/resource.py b/skew/resources/resource.py index d4af8c2..7eb57ea 100644 --- a/skew/resources/resource.py +++ b/skew/resources/resource.py @@ -77,6 +77,14 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): r = client.call('tag_resources', ResourceARNList=[arn], Tags=tags) LOG.debug('Tag ARN %s, r=%s', arn, r) + @classmethod + def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): + if hasattr(cls.Meta, 'resourcegroups_tagging') and (cls.Meta.resourcegroups_tagging): + client = skew.awsclient.get_awsclient( + 'resourcegroupstaggingapi', region, account, **kwargs) + r = client.call('untag_resources', ResourceARNList=[arn], TagsKeys=tags_keys) + LOG.debug('UnTag ARN %s, r=%s', arn, r) + def sleek(self): """ overrides datas frequently varied by a static value. From 8aace79e3da073d24a2577bb6a23149260a502ad Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Wed, 26 Dec 2018 15:03:37 +0100 Subject: [PATCH 27/62] Typo in TagKeys --- skew/resources/aws/autoscaling.py | 2 +- skew/resources/aws/cloudfront.py | 2 +- skew/resources/aws/efs.py | 2 +- skew/resources/aws/opsworks.py | 2 +- skew/resources/resource.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/skew/resources/aws/autoscaling.py b/skew/resources/aws/autoscaling.py index 0893525..199f30e 100644 --- a/skew/resources/aws/autoscaling.py +++ b/skew/resources/aws/autoscaling.py @@ -65,7 +65,7 @@ def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs) addon = dict(ResourceId=asg_name, ResourceType='auto-scaling-group', PropagateAtLaunch=False) - tags_list = [dict(Key=k, Value=str(''), **addon) for k in tags_keys] + tags_list = [dict(Key=k, **addon) for k in tags_keys] return client.call('delete_tags', Tags=tags_list) diff --git a/skew/resources/aws/cloudfront.py b/skew/resources/aws/cloudfront.py index 6e9b050..f05fc7a 100644 --- a/skew/resources/aws/cloudfront.py +++ b/skew/resources/aws/cloudfront.py @@ -48,4 +48,4 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): client = get_awsclient( cls.Meta.service, region, account, **kwargs) - return client.call('untag_resource', Resource=arn, TagsKeys=dict(Items=tags_keys)) + return client.call('untag_resource', Resource=arn, TagKeys=dict(Items=tags_keys)) diff --git a/skew/resources/aws/efs.py b/skew/resources/aws/efs.py index f706132..54f2746 100644 --- a/skew/resources/aws/efs.py +++ b/skew/resources/aws/efs.py @@ -64,6 +64,6 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): client = get_awsclient( cls.Meta.service, region, account, **kwargs) - x = client.call('delete_tags', FileSystemId=arn.split('/')[-1], TagsKeys=tags_keys) + x = client.call('delete_tags', FileSystemId=arn.split('/')[-1], TagKeys=tags_keys) print(x, arn.split('/')[-1], tags_keys) return x diff --git a/skew/resources/aws/opsworks.py b/skew/resources/aws/opsworks.py index 08441ae..6997c0e 100644 --- a/skew/resources/aws/opsworks.py +++ b/skew/resources/aws/opsworks.py @@ -56,5 +56,5 @@ def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs) # opsworks.untag_resource() supports both client = get_awsclient( cls.Meta.service, region, account, **kwargs) - r = client.call('untag_resource', ResourceArn=arn, TagsKeys=tags_keys) + r = client.call('untag_resource', ResourceArn=arn, TagKeys=tags_keys) LOG.debug('UnTag ARN %s, r=%s', arn, r) diff --git a/skew/resources/resource.py b/skew/resources/resource.py index 7eb57ea..bd4c35d 100644 --- a/skew/resources/resource.py +++ b/skew/resources/resource.py @@ -82,7 +82,7 @@ def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs) if hasattr(cls.Meta, 'resourcegroups_tagging') and (cls.Meta.resourcegroups_tagging): client = skew.awsclient.get_awsclient( 'resourcegroupstaggingapi', region, account, **kwargs) - r = client.call('untag_resources', ResourceARNList=[arn], TagsKeys=tags_keys) + r = client.call('untag_resources', ResourceARNList=[arn], TagKeys=tags_keys) LOG.debug('UnTag ARN %s, r=%s', arn, r) def sleek(self): From e4f0c494600108e85e389bebf436edd36b0f236a Mon Sep 17 00:00:00 2001 From: Christophe MORIO Date: Tue, 4 Sep 2018 12:21:12 +0200 Subject: [PATCH 28/62] Fix elb arn namespace : "elasticloadbalancing" instead of "elb" --- skew/resources/aws/elb.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/skew/resources/aws/elb.py b/skew/resources/aws/elb.py index 1938d7c..527fc22 100644 --- a/skew/resources/aws/elb.py +++ b/skew/resources/aws/elb.py @@ -31,3 +31,10 @@ class Meta(object): dimension = 'LoadBalancerName' tags_spec = ('describe_tags', 'TagDescriptions[].Tags[]', 'LoadBalancerNames', 'id') + + @property + def arn(self): + return 'arn:aws:elasticloadbalancing:%s:%s:%s/%s' % ( + self._client.region_name, + self._client.account_id, + self.resourcetype, self.id) From ff4acba0a0dd55cfea690c9e95874992ce82469c Mon Sep 17 00:00:00 2001 From: Christophe MORIO Date: Tue, 4 Sep 2018 12:22:25 +0200 Subject: [PATCH 29/62] Add support for 8 resources - aws.acm.certificate - aws.ecs.cluster - aws.ecs.task-definition - aws.efs.filesystem - aws.elbv2.loadbalancer - aws.elbv2.targetgroup - aws.ses.identity - aws.support.check --- skew/resources/__init__.py | 10 ++++- skew/resources/aws/acm.py | 38 ++++++++++++++++++ skew/resources/aws/ecs.py | 76 +++++++++++++++++++++++++++++++++++ skew/resources/aws/efs.py | 47 ++++++++++++++++++++++ skew/resources/aws/elbv2.py | 70 ++++++++++++++++++++++++++++++++ skew/resources/aws/ses.py | 44 ++++++++++++++++++++ skew/resources/aws/support.py | 37 +++++++++++++++++ 7 files changed, 321 insertions(+), 1 deletion(-) create mode 100644 skew/resources/aws/acm.py create mode 100644 skew/resources/aws/ecs.py create mode 100644 skew/resources/aws/efs.py create mode 100644 skew/resources/aws/elbv2.py create mode 100644 skew/resources/aws/ses.py create mode 100644 skew/resources/aws/support.py diff --git a/skew/resources/__init__.py b/skew/resources/__init__.py index 7906b2c..ed242c0 100644 --- a/skew/resources/__init__.py +++ b/skew/resources/__init__.py @@ -17,6 +17,7 @@ # Maps resources names as they appear in ARN's to the path name # of the Python class representing that resource. ResourceTypes = { + 'aws.acm.certificate': 'aws.acm.Certificate', 'aws.apigateway.restapis': 'aws.apigateway.RestAPI', 'aws.autoscaling.autoScalingGroup': 'aws.autoscaling.AutoScalingGroup', 'aws.autoscaling.launchConfigurationName': 'aws.autoscaling.LaunchConfiguration', @@ -40,12 +41,17 @@ 'aws.ec2.vpc-peering-connection': 'aws.ec2.VpcPeeringConnection', 'aws.ec2.subnet': 'aws.ec2.Subnet', 'aws.ec2.launch-template': 'aws.ec2.LaunchTemplate', + 'aws.ecs.cluster': 'aws.ecs.Cluster', + 'aws.ecs.task-definition': 'aws.ecs.TaskDefinition', + 'aws.efs.filesystem': 'aws.efs.Filesystem', 'aws.elasticache.cluster': 'aws.elasticache.Cluster', 'aws.elasticache.subnet-group': 'aws.elasticache.SubnetGroup', 'aws.elasticache.snapshot': 'aws.elasticache.Snapshot', 'aws.elasticbeanstalk.application': 'aws.elasticbeanstalk.Application', 'aws.elasticbeanstalk.environment': 'aws.elasticbeanstalk.Environment', 'aws.elb.loadbalancer': 'aws.elb.LoadBalancer', + 'aws.elbv2.loadbalancer': 'aws.elbv2.LoadBalancer', + 'aws.elbv2.targetgroup': 'aws.elbv2.TargetGroup', 'aws.es.domain': 'aws.es.ElasticsearchDomain', 'aws.firehose.deliverystream': 'aws.firehose.DeliveryStream', 'aws.iam.group': 'aws.iam.Group', @@ -63,8 +69,10 @@ 'aws.route53.healthcheck': 'aws.route53.HealthCheck', 'aws.s3.bucket': 'aws.s3.Bucket', 'aws.sqs.queue': 'aws.sqs.Queue', + 'aws.ses.identity': 'aws.ses.Identity', 'aws.sns.subscription': 'aws.sns.Subscription', - 'aws.sns.topic': 'aws.sns.Topic' + 'aws.sns.topic': 'aws.sns.Topic', + 'aws.support.check': 'aws.support.Check' } diff --git a/skew/resources/aws/acm.py b/skew/resources/aws/acm.py new file mode 100644 index 0000000..e6da847 --- /dev/null +++ b/skew/resources/aws/acm.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Certificate(AWSResource): + + class Meta(object): + service = 'acm' + type = 'certificate' + enum_spec = ('list_certificates', 'CertificateSummaryList', None) + detail_spec = None + id = 'CertificateArn' + tags_spec = ('list_tags_for_certificate', 'Tags[]', + 'CertificateArn', 'id') + filter_name = None + name = 'DomainName' + date = None + dimension = None diff --git a/skew/resources/aws/ecs.py b/skew/resources/aws/ecs.py new file mode 100644 index 0000000..91663dd --- /dev/null +++ b/skew/resources/aws/ecs.py @@ -0,0 +1,76 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Cluster(AWSResource): + + class Meta(object): + service = 'ecs' + type = 'cluster' + enum_spec = ('list_clusters', 'clusterArns', None) + detail_spec = ('describe_clusters', 'clusters', 'clusters[0]') + id = None + + + filter_name = None + name = 'clusterName' + date = None + dimension = None + + @property + def arn(self): + return self.data['clusterArn'] + + def __init__(self, client, data, query=None): + super(Cluster, self).__init__(client, data, query) + self._id = data + detail_op, param_name, detail_path = self.Meta.detail_spec + params = {param_name: [self.id]} + data = client.call(detail_op, **params) + self.data = jmespath.search(detail_path, data) + + +class TaskDefinition(AWSResource): + + class Meta(object): + service = 'ecs' + type = 'task-definition' + enum_spec = ('list_task_definitions', 'taskDefinitionArns', None) + detail_spec = ('describe_task_definition', 'taskDefinition', 'taskDefinition') + id = None + name = None + filter_name = None + date = None + dimension = None + + def __init__(self, client, data, query=None): + super(TaskDefinition, self).__init__(client, data, query) + self._id = data + detail_op, param_name, detail_path = self.Meta.detail_spec + params = {param_name: self.id} + data = client.call(detail_op, **params) + self.data = jmespath.search(detail_path, data) + + @property + def arn(self): + return self.data['taskDefinitionArn'] \ No newline at end of file diff --git a/skew/resources/aws/efs.py b/skew/resources/aws/efs.py new file mode 100644 index 0000000..e5c21bd --- /dev/null +++ b/skew/resources/aws/efs.py @@ -0,0 +1,47 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Filesystem(AWSResource): + + class Meta(object): + service = 'efs' + type = 'filesystem' + enum_spec = ('describe_file_systems', 'FileSystems', None) + detail_spec = None + id = 'FileSystemId' + tags_spec = ('describe_tags', 'Tags[]', + 'FileSystemId', 'id') + filter_name = None + name = 'Name' + date = 'CreationTime' + dimension = None + + @property + def arn(self): + # arn:aws:elasticfilesystem:us-east-1:123456789012:file-system-id/fs12345678 + return 'arn:aws:%s:%s:%s:%s/%s' % ( + 'elasticfilesystem', + self._client.region_name, + self._client.account_id, + 'file-system-id', self.id) diff --git a/skew/resources/aws/elbv2.py b/skew/resources/aws/elbv2.py new file mode 100644 index 0000000..efebe24 --- /dev/null +++ b/skew/resources/aws/elbv2.py @@ -0,0 +1,70 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import jmespath + +from skew.resources.aws import AWSResource + + +class LoadBalancer(AWSResource): + + class Meta(object): + service = 'elbv2' + type = 'loadbalancer' + enum_spec = ('describe_load_balancers', + 'LoadBalancers', None) + # detail_spec = None + detail_spec = ('describe_listeners', 'LoadBalancerArn', 'Listeners') + id = 'LoadBalancerArn' + filter_name = 'Names' + filter_type = 'list' + name = 'DNSName' + date = 'CreatedTime' + dimension = None + tags_spec = ('describe_tags', 'TagDescriptions[].Tags[]', + 'LoadBalancerNames', 'id') + + @property + def arn(self): + return self.data['LoadBalancerArn'] + + def __init__(self, client, data, query=None): + super(LoadBalancer, self).__init__(client, data, query) + self._id = data + detail_op, param_name, detail_path = self.Meta.detail_spec + params = {param_name: self.data['LoadBalancerArn']} + data = client.call(detail_op, **params) + self.data['Listeners'] = jmespath.search(detail_path, data) + + +class TargetGroup(AWSResource): + + class Meta(object): + service = 'elbv2' + type = 'targetgroup' + enum_spec = ('describe_target_groups', + 'TargetGroups', None) + detail_spec = None + id = 'TargetGroupArn' + filter_name = 'Names' + filter_type = 'list' + name = 'TargetGroupName' + date = 'CreatedTime' + dimension = 'LoadBalancerName' + tags_spec = ('describe_tags', 'TagDescriptions[].Tags[]', + 'LoadBalancerNames', 'id') + + @property + def arn(self): + return self.data['TargetGroupArn'] diff --git a/skew/resources/aws/ses.py b/skew/resources/aws/ses.py new file mode 100644 index 0000000..90a575a --- /dev/null +++ b/skew/resources/aws/ses.py @@ -0,0 +1,44 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Identity(AWSResource): + + class Meta(object): + service = 'ses' + type = 'identity' + enum_spec = ('list_identities', 'Identities', None) + detail_spec = ('describe_table', 'TableName', 'Table') + id = 'Identity' + tags_spec = None + filter_name = None + name = 'IdentityName' + date = None + dimension = 'IdentityName' + + @property + def arn(self): + return 'arn:aws:%s:%s:%s:%s/%s' % ( + self._client.service_name, + self._client.region_name, + self._client.account_id, self.resourcetype, self.data) diff --git a/skew/resources/aws/support.py b/skew/resources/aws/support.py new file mode 100644 index 0000000..61733e6 --- /dev/null +++ b/skew/resources/aws/support.py @@ -0,0 +1,37 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Check(AWSResource): + + class Meta(object): + service = 'support' + type = 'check' + enum_spec = ('describe_trusted_advisor_checks', 'checks', None) + detail_spec = None + id = 'id' + tags_spec = None + filter_name = None + name = 'name' + date = None + dimension = 'IdentityName' From a4046ba8ad4f887090161828a4f4e99dc195e7b1 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Thu, 13 Sep 2018 14:55:02 +0200 Subject: [PATCH 30/62] Return specific ARN --- skew/resources/aws/acm.py | 4 ++++ skew/resources/aws/route53.py | 4 ++++ skew/resources/aws/s3.py | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/skew/resources/aws/acm.py b/skew/resources/aws/acm.py index e6da847..6e9b0e9 100644 --- a/skew/resources/aws/acm.py +++ b/skew/resources/aws/acm.py @@ -36,3 +36,7 @@ class Meta(object): name = 'DomainName' date = None dimension = None + + @property + def arn(self): + return self.data['CertificateArn'] diff --git a/skew/resources/aws/route53.py b/skew/resources/aws/route53.py index 5c07a94..2874ef3 100644 --- a/skew/resources/aws/route53.py +++ b/skew/resources/aws/route53.py @@ -42,6 +42,10 @@ class Meta(object): def id(self): return self._id.split('/')[-1] + @property + def arn(self): + return 'arn:aws:route:::hostedzone/{}'.format(self.id) + class HealthCheck(Route53Resource): diff --git a/skew/resources/aws/s3.py b/skew/resources/aws/s3.py index 98a2b25..3057752 100644 --- a/skew/resources/aws/s3.py +++ b/skew/resources/aws/s3.py @@ -72,3 +72,7 @@ def __iter__(self): self._keys = jmespath.search(detail_path, data) for key in self._keys: yield key + + @property + def arn(self): + return 'arn:aws:s3:::{}'.format(self.data['Name']) From 4871e90175c9ffbdbc000f756dce35eeeabe7d49 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Thu, 13 Sep 2018 14:56:33 +0200 Subject: [PATCH 31/62] Sort a list to get consistent outputs --- skew/resources/aws/autoscaling.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/skew/resources/aws/autoscaling.py b/skew/resources/aws/autoscaling.py index 0d84ee2..78d5b6c 100644 --- a/skew/resources/aws/autoscaling.py +++ b/skew/resources/aws/autoscaling.py @@ -32,6 +32,8 @@ class Meta(object): filter_type = 'list' def __init__(self, client, data, query=None): + # Always save the list in the same order to avoid false changes detection + data['EnabledMetrics'].sort(key=lambda item:item['Metric']) super(AutoScalingGroup, self).__init__(client, data, query) self._arn_query = jmespath.compile('AutoScalingGroupARN') From c79d849cdcf40b711fcdd934c8863033289fd265 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 1 Oct 2018 10:23:19 +0200 Subject: [PATCH 32/62] Add set tag feature for ACM, ASG and CloudFront --- skew/resources/aws/acm.py | 8 ++++++++ skew/resources/aws/autoscaling.py | 12 ++++++++++++ skew/resources/aws/cloudfront.py | 8 ++++++++ skew/resources/resource.py | 4 ++++ 4 files changed, 32 insertions(+) diff --git a/skew/resources/aws/acm.py b/skew/resources/aws/acm.py index 6e9b0e9..e754739 100644 --- a/skew/resources/aws/acm.py +++ b/skew/resources/aws/acm.py @@ -17,6 +17,7 @@ import jmespath from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient LOG = logging.getLogger(__name__) @@ -40,3 +41,10 @@ class Meta(object): @property def arn(self): return self.data['CertificateArn'] + + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + tags_list = [dict(Key=k, Value=str(v)) for k, v in tags.items()] + return client.call('add_tags_to_certificate', CertificateArn=arn, Tags=tags_list) diff --git a/skew/resources/aws/autoscaling.py b/skew/resources/aws/autoscaling.py index 78d5b6c..54a191a 100644 --- a/skew/resources/aws/autoscaling.py +++ b/skew/resources/aws/autoscaling.py @@ -15,6 +15,7 @@ import jmespath from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient class AutoScalingGroup(AWSResource): @@ -41,6 +42,17 @@ def __init__(self, client, data, query=None): def arn(self): return self._arn_query.search(self.data) + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + asg_name = arn.split(':')[7].split('/')[1] + addon = dict(ResourceId=asg_name, + ResourceType='auto-scaling-group', + PropagateAtLaunch=False) + tags_list = [dict(Key=k, Value=str(v), **addon) for k, v in tags.items()] + return client.call('create_or_update_tags', Tags=tags_list) + class LaunchConfiguration(AWSResource): diff --git a/skew/resources/aws/cloudfront.py b/skew/resources/aws/cloudfront.py index 5b096fa..113efdb 100644 --- a/skew/resources/aws/cloudfront.py +++ b/skew/resources/aws/cloudfront.py @@ -1,6 +1,7 @@ import logging from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient LOG = logging.getLogger(__name__) @@ -34,3 +35,10 @@ class Meta(object): def filter(cls, arn, resource_id, data): LOG.debug('%s == %s', resource_id, data) return resource_id == data['Id'] + + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + tags_list = [dict(Key=k, Value=str(v)) for k, v in tags.items()] + return client.call('tag_resource', Resource=arn, Tags=dict(Items=tags_list)) diff --git a/skew/resources/resource.py b/skew/resources/resource.py index 92514cd..d1a4af0 100644 --- a/skew/resources/resource.py +++ b/skew/resources/resource.py @@ -69,6 +69,10 @@ def enumerate(cls, arn, region, account, resource_id=None, **kwargs): resources.append(cls(client, d, arn.query)) return resources + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + pass + class Meta(object): type = 'resource' dimension = None From a9701ca38a2388f249643ec25714882ac588f089 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Wed, 3 Oct 2018 14:06:17 +0200 Subject: [PATCH 33/62] Set the Name from the instance ID, because PrivateDnsName count be undefined, and PrivateIpAddress could be not unique --- skew/resources/aws/ec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skew/resources/aws/ec2.py b/skew/resources/aws/ec2.py index 18253ad..6b48717 100644 --- a/skew/resources/aws/ec2.py +++ b/skew/resources/aws/ec2.py @@ -25,7 +25,7 @@ class Meta(object): id = 'InstanceId' filter_name = 'InstanceIds' filter_type = 'list' - name = 'PublicDnsName' + name = 'InstanceId' date = 'LaunchTime' dimension = 'InstanceId' From 74b43a29b36e1cde30a0d51cc159476ad5ed0836 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Wed, 3 Oct 2018 17:26:08 +0200 Subject: [PATCH 34/62] Define asset name from short Id/Name instead of fqdn --- skew/resources/aws/cloudfront.py | 2 +- skew/resources/aws/elb.py | 2 +- skew/resources/aws/elbv2.py | 2 +- skew/resources/aws/sqs.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/skew/resources/aws/cloudfront.py b/skew/resources/aws/cloudfront.py index 113efdb..865f950 100644 --- a/skew/resources/aws/cloudfront.py +++ b/skew/resources/aws/cloudfront.py @@ -26,7 +26,7 @@ class Meta(object): id = 'Id' tags_spec = ('list_tags_for_resource', 'Tags.Items[]', 'Resource', 'arn') - name = 'DomainName' + name = 'Id' filter_name = None date = 'LastModifiedTime' dimension = None diff --git a/skew/resources/aws/elb.py b/skew/resources/aws/elb.py index 527fc22..5381c4f 100644 --- a/skew/resources/aws/elb.py +++ b/skew/resources/aws/elb.py @@ -26,7 +26,7 @@ class Meta(object): id = 'LoadBalancerName' filter_name = 'LoadBalancerNames' filter_type = 'list' - name = 'DNSName' + name = 'LoadBalancerName' date = 'CreatedTime' dimension = 'LoadBalancerName' tags_spec = ('describe_tags', 'TagDescriptions[].Tags[]', diff --git a/skew/resources/aws/elbv2.py b/skew/resources/aws/elbv2.py index efebe24..72382ff 100644 --- a/skew/resources/aws/elbv2.py +++ b/skew/resources/aws/elbv2.py @@ -29,7 +29,7 @@ class Meta(object): id = 'LoadBalancerArn' filter_name = 'Names' filter_type = 'list' - name = 'DNSName' + name = 'LoadBalancerName' date = 'CreatedTime' dimension = None tags_spec = ('describe_tags', 'TagDescriptions[].Tags[]', diff --git a/skew/resources/aws/sqs.py b/skew/resources/aws/sqs.py index 9517bcb..4f22e07 100644 --- a/skew/resources/aws/sqs.py +++ b/skew/resources/aws/sqs.py @@ -25,7 +25,7 @@ class Meta(object): id = 'QueueUrl' filter_name = 'QueueNamePrefix' filter_type = 'scalar' - name = 'QueueUrl' + name = 'QueueName' date = None dimension = 'QueueName' From a2019fa90afd6846aa4ea8c530b2207d123bab5c Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 15 Oct 2018 12:07:37 +0200 Subject: [PATCH 35/62] Prefer short name --- skew/resources/aws/rds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skew/resources/aws/rds.py b/skew/resources/aws/rds.py index e943766..0b2c29b 100644 --- a/skew/resources/aws/rds.py +++ b/skew/resources/aws/rds.py @@ -27,7 +27,7 @@ class Meta(object): id = 'DBInstanceIdentifier' filter_name = 'DBInstanceIdentifier' filter_type = 'scalar' - name = 'Endpoint.Address' + name = 'DBInstanceIdentifier' date = 'InstanceCreateTime' dimension = 'DBInstanceIdentifier' From b4ec71fcf09a5e09e95725e6e55558c8420e7943 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 15 Oct 2018 12:08:02 +0200 Subject: [PATCH 36/62] Fix tags read --- skew/resources/aws/elbv2.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/skew/resources/aws/elbv2.py b/skew/resources/aws/elbv2.py index 72382ff..c9ebc67 100644 --- a/skew/resources/aws/elbv2.py +++ b/skew/resources/aws/elbv2.py @@ -33,7 +33,7 @@ class Meta(object): date = 'CreatedTime' dimension = None tags_spec = ('describe_tags', 'TagDescriptions[].Tags[]', - 'LoadBalancerNames', 'id') + 'ResourceArns', 'id') @property def arn(self): @@ -41,7 +41,6 @@ def arn(self): def __init__(self, client, data, query=None): super(LoadBalancer, self).__init__(client, data, query) - self._id = data detail_op, param_name, detail_path = self.Meta.detail_spec params = {param_name: self.data['LoadBalancerArn']} data = client.call(detail_op, **params) From 7d232563d45a01a684a724e2716aa51c71e780dd Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 15 Oct 2018 12:12:54 +0200 Subject: [PATCH 37/62] Class method for tag writing --- skew/resources/aws/__init__.py | 1 + skew/resources/resource.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/skew/resources/aws/__init__.py b/skew/resources/aws/__init__.py index d3dd49c..496b839 100644 --- a/skew/resources/aws/__init__.py +++ b/skew/resources/aws/__init__.py @@ -47,6 +47,7 @@ class AWSResource(Resource): Each entry in the dictionary we define: * service - The AWS service in which this resource is defined. + * tags_support - Precise if the resource supports tags. * enum_spec - The enumeration configuration. This is a tuple consisting of the name of the operation to call to enumerate the resources, a jmespath query that will be run against the result of the operation diff --git a/skew/resources/resource.py b/skew/resources/resource.py index d1a4af0..b3ac6f8 100644 --- a/skew/resources/resource.py +++ b/skew/resources/resource.py @@ -71,7 +71,10 @@ def enumerate(cls, arn, region, account, resource_id=None, **kwargs): @classmethod def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): - pass + if hasattr(cls.Meta, 'tags_support') and (cls.Meta.tags_support): + client = skew.awsclient.get_awsclient( + 'resourcegroupstaggingapi', region, account, **kwargs) + client.call('tag_resources', ResourceARNList=[arn], Tags=tags) class Meta(object): type = 'resource' From b9ee73e256d6d061c1e3444d50a82d5eb4f546b5 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 15 Oct 2018 17:50:03 +0200 Subject: [PATCH 38/62] Fix the ARN with specific convention --- skew/resources/aws/sqs.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/skew/resources/aws/sqs.py b/skew/resources/aws/sqs.py index 4f22e07..c4b4c90 100644 --- a/skew/resources/aws/sqs.py +++ b/skew/resources/aws/sqs.py @@ -34,3 +34,9 @@ def __init__(self, client, data, query=None): self.data = {self.Meta.id: data, 'QueueName': data.split('/')[-1]} self._id = self.data['QueueName'] + + @property + def arn(self): + return 'arn:aws:sqs:%s:%s:%s' % ( + self._client.region_name, + self._client.account_id, self.id) \ No newline at end of file From e60c818467068b4525c2d0356ccd99d820841e02 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 15 Oct 2018 17:52:01 +0200 Subject: [PATCH 39/62] Rename tags_support to resourcegroups_tagging --- skew/resources/aws/__init__.py | 2 +- skew/resources/resource.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/skew/resources/aws/__init__.py b/skew/resources/aws/__init__.py index 496b839..28adb8a 100644 --- a/skew/resources/aws/__init__.py +++ b/skew/resources/aws/__init__.py @@ -47,7 +47,7 @@ class AWSResource(Resource): Each entry in the dictionary we define: * service - The AWS service in which this resource is defined. - * tags_support - Precise if the resource supports tags. + * resourcegroups_tagging - Precise if the resource supports tags. * enum_spec - The enumeration configuration. This is a tuple consisting of the name of the operation to call to enumerate the resources, a jmespath query that will be run against the result of the operation diff --git a/skew/resources/resource.py b/skew/resources/resource.py index b3ac6f8..bb12171 100644 --- a/skew/resources/resource.py +++ b/skew/resources/resource.py @@ -71,10 +71,11 @@ def enumerate(cls, arn, region, account, resource_id=None, **kwargs): @classmethod def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): - if hasattr(cls.Meta, 'tags_support') and (cls.Meta.tags_support): + if hasattr(cls.Meta, 'resourcegroups_tagging') and (cls.Meta.resourcegroups_tagging): client = skew.awsclient.get_awsclient( 'resourcegroupstaggingapi', region, account, **kwargs) - client.call('tag_resources', ResourceARNList=[arn], Tags=tags) + r = client.call('tag_resources', ResourceARNList=[arn], Tags=tags) + logging.warn('Tag ARN %s : %s', arn, r) class Meta(object): type = 'resource' From 458bd7999cedc8589d84daed3bd68d5dbb45aa33 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 15 Oct 2018 17:52:17 +0200 Subject: [PATCH 40/62] Enable Resource Groups Tagging (write) when possible --- skew/resources/aws/acm.py | 8 +------- skew/resources/aws/apigateway.py | 1 + skew/resources/aws/autoscaling.py | 1 + skew/resources/aws/cloudformation.py | 1 + skew/resources/aws/cloudfront.py | 1 + skew/resources/aws/cloudwatch.py | 1 + skew/resources/aws/dynamodb.py | 1 + skew/resources/aws/ec2.py | 14 ++++++++++++++ skew/resources/aws/ecs.py | 2 ++ skew/resources/aws/efs.py | 1 + skew/resources/aws/elasticache.py | 3 +++ skew/resources/aws/elasticbeanstalk.py | 8 +++++++- skew/resources/aws/elb.py | 1 + skew/resources/aws/elbv2.py | 2 ++ skew/resources/aws/es.py | 1 + skew/resources/aws/firehose.py | 1 + skew/resources/aws/iam.py | 6 ++++++ skew/resources/aws/kinesis.py | 1 + skew/resources/aws/lambda.py | 1 + skew/resources/aws/rds.py | 2 ++ skew/resources/aws/redshift.py | 1 + skew/resources/aws/route53.py | 3 +++ skew/resources/aws/s3.py | 1 + skew/resources/aws/ses.py | 1 + skew/resources/aws/sns.py | 2 ++ skew/resources/aws/sqs.py | 1 + skew/resources/aws/support.py | 1 + 27 files changed, 59 insertions(+), 8 deletions(-) diff --git a/skew/resources/aws/acm.py b/skew/resources/aws/acm.py index e754739..8402651 100644 --- a/skew/resources/aws/acm.py +++ b/skew/resources/aws/acm.py @@ -28,6 +28,7 @@ class Certificate(AWSResource): class Meta(object): service = 'acm' type = 'certificate' + resourcegroups_tagging = True enum_spec = ('list_certificates', 'CertificateSummaryList', None) detail_spec = None id = 'CertificateArn' @@ -41,10 +42,3 @@ class Meta(object): @property def arn(self): return self.data['CertificateArn'] - - @classmethod - def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): - client = get_awsclient( - cls.Meta.service, region, account, **kwargs) - tags_list = [dict(Key=k, Value=str(v)) for k, v in tags.items()] - return client.call('add_tags_to_certificate', CertificateArn=arn, Tags=tags_list) diff --git a/skew/resources/aws/apigateway.py b/skew/resources/aws/apigateway.py index a460a0a..0601868 100644 --- a/skew/resources/aws/apigateway.py +++ b/skew/resources/aws/apigateway.py @@ -21,6 +21,7 @@ class RestAPI(AWSResource): class Meta(object): service = 'apigateway' type = 'restapis' + resourcegroups_tagging = False enum_spec = ('get_rest_apis', 'items', None) id = 'id' filter_name = None diff --git a/skew/resources/aws/autoscaling.py b/skew/resources/aws/autoscaling.py index 54a191a..31d4385 100644 --- a/skew/resources/aws/autoscaling.py +++ b/skew/resources/aws/autoscaling.py @@ -23,6 +23,7 @@ class AutoScalingGroup(AWSResource): class Meta(object): service = 'autoscaling' type = 'autoScalingGroup' + resourcegroups_tagging = False name = 'AutoScalingGroupName' date = 'CreatedTime' dimension = 'AutoScalingGroupName' diff --git a/skew/resources/aws/cloudformation.py b/skew/resources/aws/cloudformation.py index 782af9f..96bc1e5 100644 --- a/skew/resources/aws/cloudformation.py +++ b/skew/resources/aws/cloudformation.py @@ -38,6 +38,7 @@ def enumerate(cls, arn, region, account, resource_id=None, **kwargs): class Meta(object): service = 'cloudformation' type = 'stack' + resourcegroups_tagging = False enum_spec = ('describe_stacks', 'Stacks[]', None) detail_spec = ('describe_stack_resources', 'StackName', 'StackResources[]') diff --git a/skew/resources/aws/cloudfront.py b/skew/resources/aws/cloudfront.py index 865f950..61f4793 100644 --- a/skew/resources/aws/cloudfront.py +++ b/skew/resources/aws/cloudfront.py @@ -21,6 +21,7 @@ class Distribution(CloudfrontResource): class Meta(object): service = 'cloudfront' type = 'distribution' + resourcegroups_tagging = False enum_spec = ('list_distributions', 'DistributionList.Items[]', None) detail_spec = None id = 'Id' diff --git a/skew/resources/aws/cloudwatch.py b/skew/resources/aws/cloudwatch.py index 89807bb..69916d0 100644 --- a/skew/resources/aws/cloudwatch.py +++ b/skew/resources/aws/cloudwatch.py @@ -20,6 +20,7 @@ class Alarm(AWSResource): class Meta(object): service = 'cloudwatch' type = 'alarm' + resourcegroups_tagging = False enum_spec = ('describe_alarms', 'MetricAlarms', None) id = 'AlarmArn' filter_name = 'AlarmNames' diff --git a/skew/resources/aws/dynamodb.py b/skew/resources/aws/dynamodb.py index 196c6b7..074ed69 100644 --- a/skew/resources/aws/dynamodb.py +++ b/skew/resources/aws/dynamodb.py @@ -27,6 +27,7 @@ class Table(AWSResource): class Meta(object): service = 'dynamodb' type = 'table' + resourcegroups_tagging = True enum_spec = ('list_tables', 'TableNames', None) detail_spec = ('describe_table', 'TableName', 'Table') id = 'Table' diff --git a/skew/resources/aws/ec2.py b/skew/resources/aws/ec2.py index 6b48717..3756208 100644 --- a/skew/resources/aws/ec2.py +++ b/skew/resources/aws/ec2.py @@ -20,6 +20,7 @@ class Instance(AWSResource): class Meta(object): service = 'ec2' type = 'instance' + resourcegroups_tagging = True enum_spec = ('describe_instances', 'Reservations[].Instances[]', None) detail_spec = None id = 'InstanceId' @@ -39,6 +40,7 @@ class SecurityGroup(AWSResource): class Meta(object): service = 'ec2' type = 'security-group' + resourcegroups_tagging = True enum_spec = ('describe_security_groups', 'SecurityGroups', None) detail_spec = None id = 'GroupId' @@ -54,6 +56,7 @@ class KeyPair(AWSResource): class Meta(object): service = 'ec2' type = 'key-pair' + resourcegroups_tagging = False enum_spec = ('describe_key_pairs', 'KeyPairs', None) detail_spec = None id = 'KeyName' @@ -68,6 +71,7 @@ class Address(AWSResource): class Meta(object): service = 'ec2' type = 'address' + resourcegroups_tagging = False enum_spec = ('describe_addresses', 'Addresses', None) detail_spec = None id = 'PublicIp' @@ -83,6 +87,7 @@ class Volume(AWSResource): class Meta(object): service = 'ec2' type = 'volume' + resourcegroups_tagging = True enum_spec = ('describe_volumes', 'Volumes', None) detail_spec = None id = 'VolumeId' @@ -105,6 +110,7 @@ class Snapshot(AWSResource): class Meta(object): service = 'ec2' type = 'snapshot' + resourcegroups_tagging = True enum_spec = ( 'describe_snapshots', 'Snapshots', {'OwnerIds': ['self']}) detail_spec = None @@ -128,6 +134,7 @@ class Image(AWSResource): class Meta(object): service = 'ec2' type = 'image' + resourcegroups_tagging = True enum_spec = ( 'describe_images', 'Images', {'Owners': ['self']}) detail_spec = None @@ -151,6 +158,7 @@ class Vpc(AWSResource): class Meta(object): service = 'ec2' type = 'vpc' + resourcegroups_tagging = True enum_spec = ('describe_vpcs', 'Vpcs', None) detail_spec = None id = 'VpcId' @@ -166,6 +174,7 @@ class Subnet(AWSResource): class Meta(object): service = 'ec2' type = 'subnet' + resourcegroups_tagging = True enum_spec = ('describe_subnets', 'Subnets', None) detail_spec = None id = 'SubnetId' @@ -181,6 +190,7 @@ class CustomerGateway(AWSResource): class Meta(object): service = 'ec2' type = 'customer-gateway' + resourcegroups_tagging = True enum_spec = ('describe_customer_gateways', 'CustomerGateway', None) detail_spec = None id = 'CustomerGatewayId' @@ -197,6 +207,7 @@ class Meta(object): service = 'ec2' type = 'internet-gateway' enum_spec = ('describe_internet_gateways', 'InternetGateways', None) + resourcegroups_tagging = True detail_spec = None id = 'InternetGatewayId' filter_name = 'InternetGatewayIds' @@ -211,6 +222,7 @@ class RouteTable(AWSResource): class Meta(object): service = 'ec2' type = 'route-table' + resourcegroups_tagging = True enum_spec = ('describe_route_tables', 'RouteTables', None) detail_spec = None id = 'RouteTableId' @@ -240,6 +252,7 @@ class NetworkAcl(AWSResource): class Meta(object): service = 'ec2' type = 'network-acl' + resourcegroups_tagging = True enum_spec = ('describe_network_acls', 'NetworkAcls', None) detail_spec = None id = 'NetworkAclId' @@ -255,6 +268,7 @@ class VpcPeeringConnection(AWSResource): class Meta(object): service = 'ec2' type = 'vpc-peering-connection' + resourcegroups_tagging = False enum_spec = ('describe_vpc_peering_connections', 'VpcPeeringConnection', None) detail_spec = None diff --git a/skew/resources/aws/ecs.py b/skew/resources/aws/ecs.py index 91663dd..7927755 100644 --- a/skew/resources/aws/ecs.py +++ b/skew/resources/aws/ecs.py @@ -27,6 +27,7 @@ class Cluster(AWSResource): class Meta(object): service = 'ecs' type = 'cluster' + resourcegroups_tagging = False enum_spec = ('list_clusters', 'clusterArns', None) detail_spec = ('describe_clusters', 'clusters', 'clusters[0]') id = None @@ -55,6 +56,7 @@ class TaskDefinition(AWSResource): class Meta(object): service = 'ecs' type = 'task-definition' + resourcegroups_tagging = False enum_spec = ('list_task_definitions', 'taskDefinitionArns', None) detail_spec = ('describe_task_definition', 'taskDefinition', 'taskDefinition') id = None diff --git a/skew/resources/aws/efs.py b/skew/resources/aws/efs.py index e5c21bd..38d87ba 100644 --- a/skew/resources/aws/efs.py +++ b/skew/resources/aws/efs.py @@ -27,6 +27,7 @@ class Filesystem(AWSResource): class Meta(object): service = 'efs' type = 'filesystem' + resourcegroups_tagging = False enum_spec = ('describe_file_systems', 'FileSystems', None) detail_spec = None id = 'FileSystemId' diff --git a/skew/resources/aws/elasticache.py b/skew/resources/aws/elasticache.py index bbfbfc9..7f5141f 100644 --- a/skew/resources/aws/elasticache.py +++ b/skew/resources/aws/elasticache.py @@ -20,6 +20,7 @@ class Cluster(AWSResource): class Meta(object): service = 'elasticache' type = 'cluster' + resourcegroups_tagging = True enum_spec = ('describe_cache_clusters', 'CacheClusters[]', None) detail_spec = None @@ -45,6 +46,7 @@ class SubnetGroup(AWSResource): class Meta(object): service = 'elasticache' type = 'subnet-group' + resourcegroups_tagging = False enum_spec = ('describe_cache_subnet_groups', 'CacheSubnetGroups', None) detail_spec = None @@ -61,6 +63,7 @@ class Snapshot(AWSResource): class Meta(object): service = 'elasticache' type = 'snapshot' + resourcegroups_tagging = True enum_spec = ('describe_snapshots', 'Snapshots', None) detail_spec = None id = 'SnapshotName' diff --git a/skew/resources/aws/elasticbeanstalk.py b/skew/resources/aws/elasticbeanstalk.py index be2e598..7b8c31a 100644 --- a/skew/resources/aws/elasticbeanstalk.py +++ b/skew/resources/aws/elasticbeanstalk.py @@ -17,6 +17,7 @@ class Application(AWSResource): class Meta(object): service = 'elasticbeanstalk' type = 'application' + resourcegroups_tagging = False enum_spec = ('describe_applications', 'Applications', None) detail_spec = None id = 'ApplicationName' @@ -31,6 +32,7 @@ class Environment(AWSResource): class Meta(object): service = 'elasticbeanstalk' type = 'environment' + resourcegroups_tagging = True enum_spec = ('describe_environments', 'Environments', None) detail_spec = None id = 'EnvironmentName' @@ -38,4 +40,8 @@ class Meta(object): filter_type = None name = 'EnvironmentName' date = None - dimension = None \ No newline at end of file + dimension = None + + @property + def arn(self): + return self.data['EnvironmentArn'] diff --git a/skew/resources/aws/elb.py b/skew/resources/aws/elb.py index 5381c4f..c03db69 100644 --- a/skew/resources/aws/elb.py +++ b/skew/resources/aws/elb.py @@ -20,6 +20,7 @@ class LoadBalancer(AWSResource): class Meta(object): service = 'elb' type = 'loadbalancer' + resourcegroups_tagging = True enum_spec = ('describe_load_balancers', 'LoadBalancerDescriptions', None) detail_spec = None diff --git a/skew/resources/aws/elbv2.py b/skew/resources/aws/elbv2.py index c9ebc67..bb2d4de 100644 --- a/skew/resources/aws/elbv2.py +++ b/skew/resources/aws/elbv2.py @@ -22,6 +22,7 @@ class LoadBalancer(AWSResource): class Meta(object): service = 'elbv2' type = 'loadbalancer' + resourcegroups_tagging = True enum_spec = ('describe_load_balancers', 'LoadBalancers', None) # detail_spec = None @@ -52,6 +53,7 @@ class TargetGroup(AWSResource): class Meta(object): service = 'elbv2' type = 'targetgroup' + resourcegroups_tagging = False enum_spec = ('describe_target_groups', 'TargetGroups', None) detail_spec = None diff --git a/skew/resources/aws/es.py b/skew/resources/aws/es.py index 3243ebf..8a13247 100644 --- a/skew/resources/aws/es.py +++ b/skew/resources/aws/es.py @@ -22,6 +22,7 @@ class ElasticsearchDomain(AWSResource): class Meta(object): service = 'es' type = 'domain' + resourcegroups_tagging = True enum_spec = ('list_domain_names', 'DomainNames[].DomainName', None) tags_spec = ('list_tags', 'TagList', 'ARN', 'arn') diff --git a/skew/resources/aws/firehose.py b/skew/resources/aws/firehose.py index f498b9d..103f7dc 100644 --- a/skew/resources/aws/firehose.py +++ b/skew/resources/aws/firehose.py @@ -18,6 +18,7 @@ class DeliveryStream(AWSResource): class Meta(object): service = 'firehose' type = 'deliverystream' + resourcegroups_tagging = False enum_spec = ('list_delivery_streams', 'DeliveryStreamNames', None) detail_spec = ('describe_delivery_stream', 'DeliveryStreamName', 'DeliveryStreamDescription') id = 'DeliveryStreamName' diff --git a/skew/resources/aws/iam.py b/skew/resources/aws/iam.py index 3485640..cd70514 100644 --- a/skew/resources/aws/iam.py +++ b/skew/resources/aws/iam.py @@ -34,6 +34,7 @@ class Group(IAMResource): class Meta(object): service = 'iam' type = 'group' + resourcegroups_tagging = False enum_spec = ('list_groups', 'Groups', None) detail_spec = None id = 'GroupId' @@ -53,6 +54,7 @@ class User(IAMResource): class Meta(object): service = 'iam' type = 'user' + resourcegroups_tagging = False enum_spec = ('list_users', 'Users', None) detail_spec = None id = 'UserId' @@ -72,6 +74,7 @@ class Role(IAMResource): class Meta(object): service = 'iam' type = 'role' + resourcegroups_tagging = False enum_spec = ('list_roles', 'Roles', None) detail_spec = None id = 'RoleId' @@ -91,6 +94,7 @@ class InstanceProfile(IAMResource): class Meta(object): service = 'iam' type = 'instance-profile' + resourcegroups_tagging = False enum_spec = ('list_instance_profiles', 'InstanceProfiles', None) detail_spec = None id = 'InstanceProfileId' @@ -110,6 +114,7 @@ class Policy(IAMResource): class Meta(object): service = 'iam' type = 'policy' + resourcegroups_tagging = False enum_spec = ('list_policies', 'Policies', None) detail_spec = None id = 'PolicyId' @@ -129,6 +134,7 @@ class ServerCertificate(IAMResource): class Meta(object): service = 'iam' type = 'server-certificate' + resourcegroups_tagging = False enum_spec = ('list_server_certificates', 'ServerCertificateMetadataList', None) diff --git a/skew/resources/aws/kinesis.py b/skew/resources/aws/kinesis.py index 69f009c..4d6f4b3 100644 --- a/skew/resources/aws/kinesis.py +++ b/skew/resources/aws/kinesis.py @@ -20,6 +20,7 @@ class Stream(AWSResource): class Meta(object): service = 'kinesis' type = 'stream' + resourcegroups_tagging = True enum_spec = ('list_streams', 'StreamNames', None) detail_spec = None id = 'StreamName' diff --git a/skew/resources/aws/lambda.py b/skew/resources/aws/lambda.py index c42cc87..b3cfe4d 100644 --- a/skew/resources/aws/lambda.py +++ b/skew/resources/aws/lambda.py @@ -37,6 +37,7 @@ def enumerate(cls, arn, region, account, resource_id=None, **kwargs): class Meta(object): service = 'lambda' type = 'function' + resourcegroups_tagging = True enum_spec = ('list_functions', 'Functions', None) detail_spec = None id = 'FunctionName' diff --git a/skew/resources/aws/rds.py b/skew/resources/aws/rds.py index 0b2c29b..6a08eaa 100644 --- a/skew/resources/aws/rds.py +++ b/skew/resources/aws/rds.py @@ -20,6 +20,7 @@ class DBInstance(AWSResource): class Meta(object): service = 'rds' type = 'db' + resourcegroups_tagging = True enum_spec = ('describe_db_instances', 'DBInstances', None) tags_spec = ('list_tags_for_resource', 'TagList', 'ResourceName', 'arn') @@ -44,6 +45,7 @@ class DBSecurityGroup(AWSResource): class Meta(object): service = 'rds' type = 'secgrp' + resourcegroups_tagging = True enum_spec = ('describe_db_security_groups', 'DBSecurityGroups', None) detail_spec = None id = 'DBSecurityGroupName' diff --git a/skew/resources/aws/redshift.py b/skew/resources/aws/redshift.py index a39430e..07e974d 100644 --- a/skew/resources/aws/redshift.py +++ b/skew/resources/aws/redshift.py @@ -20,6 +20,7 @@ class Cluster(AWSResource): class Meta(object): service = 'redshift' type = 'cluster' + resourcegroups_tagging = True enum_spec = ('describe_clusters', 'Clusters', None) detail_spec = None id = 'ClusterIdentifier' diff --git a/skew/resources/aws/route53.py b/skew/resources/aws/route53.py index 2874ef3..21460fe 100644 --- a/skew/resources/aws/route53.py +++ b/skew/resources/aws/route53.py @@ -28,6 +28,7 @@ class HostedZone(Route53Resource): class Meta(object): service = 'route53' type = 'hostedzone' + resourcegroups_tagging = True enum_spec = ('list_hosted_zones', 'HostedZones', None) detail_spec = ('GetHostedZone', 'Id', None) id = 'Id' @@ -52,6 +53,7 @@ class HealthCheck(Route53Resource): class Meta(object): service = 'route53' type = 'healthcheck' + resourcegroups_tagging = True enum_spec = ('list_health_checks', 'HealthChecks', None) detail_spec = ('GetHealthCheck', 'Id', None) id = 'Id' @@ -68,6 +70,7 @@ class ResourceRecordSet(Route53Resource): class Meta(object): service = 'route53' type = 'rrset' + resourcegroups_tagging = False enum_spec = ('list_resource_record_sets', 'ResourceRecordSets', None) detail_spec = None id = 'Name' diff --git a/skew/resources/aws/s3.py b/skew/resources/aws/s3.py index 3057752..0b22922 100644 --- a/skew/resources/aws/s3.py +++ b/skew/resources/aws/s3.py @@ -49,6 +49,7 @@ def enumerate(cls, arn, region, account, resource_id=None, **kwargs): class Meta(object): service = 's3' type = 'bucket' + resourcegroups_tagging = True enum_spec = ('list_buckets', 'Buckets[]', None) detail_spec = ('list_objects', 'Bucket', 'Contents[]') id = 'Name' diff --git a/skew/resources/aws/ses.py b/skew/resources/aws/ses.py index 90a575a..ec7608c 100644 --- a/skew/resources/aws/ses.py +++ b/skew/resources/aws/ses.py @@ -27,6 +27,7 @@ class Identity(AWSResource): class Meta(object): service = 'ses' type = 'identity' + resourcegroups_tagging = False enum_spec = ('list_identities', 'Identities', None) detail_spec = ('describe_table', 'TableName', 'Table') id = 'Identity' diff --git a/skew/resources/aws/sns.py b/skew/resources/aws/sns.py index cf0c8ef..216e6d6 100644 --- a/skew/resources/aws/sns.py +++ b/skew/resources/aws/sns.py @@ -23,6 +23,7 @@ class Topic(AWSResource): class Meta(object): service = 'sns' type = 'topic' + resourcegroups_tagging = False enum_spec = ('list_topics', 'Topics', None) detail_spec = ('get_topic_attributes', 'TopicArn', 'Attributes') id = 'TopicArn' @@ -61,6 +62,7 @@ class Subscription(AWSResource): class Meta(object): service = 'sns' type = 'subscription' + resourcegroups_tagging = False enum_spec = ('list_subscriptions', 'Subscriptions', None) detail_spec = ('get_subscription_attributes', 'SubscriptionArn', 'Attributes') diff --git a/skew/resources/aws/sqs.py b/skew/resources/aws/sqs.py index c4b4c90..4f6e5af 100644 --- a/skew/resources/aws/sqs.py +++ b/skew/resources/aws/sqs.py @@ -20,6 +20,7 @@ class Queue(AWSResource): class Meta(object): service = 'sqs' type = 'queue' + resourcegroups_tagging = True enum_spec = ('list_queues', 'QueueUrls', None) detail_spec = ('get_queue_attributes', 'QueueUrl', 'QueueUrl') id = 'QueueUrl' diff --git a/skew/resources/aws/support.py b/skew/resources/aws/support.py index 61733e6..d957140 100644 --- a/skew/resources/aws/support.py +++ b/skew/resources/aws/support.py @@ -27,6 +27,7 @@ class Check(AWSResource): class Meta(object): service = 'support' type = 'check' + resourcegroups_tagging = False enum_spec = ('describe_trusted_advisor_checks', 'checks', None) detail_spec = None id = 'id' From cada9c0c33c7ee706528e5032754b4f735bfbea9 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 22 Oct 2018 17:05:38 +0200 Subject: [PATCH 41/62] Sleek datas frequently varied --- skew/resources/aws/efs.py | 4 ++++ skew/resources/aws/rds.py | 2 ++ skew/resources/resource.py | 6 ++++++ 3 files changed, 12 insertions(+) diff --git a/skew/resources/aws/efs.py b/skew/resources/aws/efs.py index 38d87ba..f2c9d72 100644 --- a/skew/resources/aws/efs.py +++ b/skew/resources/aws/efs.py @@ -17,6 +17,7 @@ import jmespath from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient LOG = logging.getLogger(__name__) @@ -46,3 +47,6 @@ def arn(self): self._client.region_name, self._client.account_id, 'file-system-id', self.id) + + def sleek(self): + self.data['SizeInBytes'] = 0 diff --git a/skew/resources/aws/rds.py b/skew/resources/aws/rds.py index 6a08eaa..b6bcf96 100644 --- a/skew/resources/aws/rds.py +++ b/skew/resources/aws/rds.py @@ -62,3 +62,5 @@ def arn(self): self._client.region_name, self._client.account_id, self.resourcetype, self.id) + def sleek(self): + self.data['LatestRestorableTime'] = '' diff --git a/skew/resources/resource.py b/skew/resources/resource.py index bb12171..b3b9c3c 100644 --- a/skew/resources/resource.py +++ b/skew/resources/resource.py @@ -77,6 +77,12 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): r = client.call('tag_resources', ResourceARNList=[arn], Tags=tags) logging.warn('Tag ARN %s : %s', arn, r) + def sleek(self): + """ + overrides datas frequently varied by a static value. + """ + pass + class Meta(object): type = 'resource' dimension = None From fae1e0e4f9efe29d8346e02c1e9db87bbb40b014 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Mon, 22 Oct 2018 17:07:13 +0200 Subject: [PATCH 42/62] Add set tag feature for EFS --- skew/resources/aws/efs.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/skew/resources/aws/efs.py b/skew/resources/aws/efs.py index f2c9d72..58e9ac0 100644 --- a/skew/resources/aws/efs.py +++ b/skew/resources/aws/efs.py @@ -50,3 +50,12 @@ def arn(self): def sleek(self): self.data['SizeInBytes'] = 0 + + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + tags_list = [dict(Key=k, Value=str(v)) for k, v in tags.items()] + x = client.call('create_tags', FileSystemId=arn.split('/')[-1], Tags=tags_list) + print(x, arn.split('/')[-1], tags_list) + return x From b0595ecc7b02c1c30b1a0ab3caa03e9a2a64d581 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Tue, 23 Oct 2018 10:46:38 +0200 Subject: [PATCH 43/62] Add OpsWorks stack --- skew/resources/__init__.py | 1 + skew/resources/aws/opsworks.py | 41 ++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 skew/resources/aws/opsworks.py diff --git a/skew/resources/__init__.py b/skew/resources/__init__.py index ed242c0..5dd233e 100644 --- a/skew/resources/__init__.py +++ b/skew/resources/__init__.py @@ -62,6 +62,7 @@ 'aws.iam.server-certificate': 'aws.iam.ServerCertificate', 'aws.kinesis.stream': 'aws.kinesis.Stream', 'aws.lambda.function': 'aws.lambda.Function', + 'aws.opsworks.stack': 'aws.opsworks.Stack', 'aws.rds.db': 'aws.rds.DBInstance', 'aws.rds.secgrp': 'aws.rds.DBSecurityGroup', 'aws.redshift.cluster': 'aws.redshift.Cluster', diff --git a/skew/resources/aws/opsworks.py b/skew/resources/aws/opsworks.py new file mode 100644 index 0000000..60b4b79 --- /dev/null +++ b/skew/resources/aws/opsworks.py @@ -0,0 +1,41 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Stack(AWSResource): + + class Meta(object): + service = 'opsworks' + type = 'stack' + resourcegroups_tagging = True # only support regional, not classic (us-east-1) + enum_spec = ('describe_stacks', 'Stacks', None) + detail_spec = None + id = 'StackId' + filter_name = None + name = 'Name' + date = 'CreatedAt' + dimension = None + tags_spec = ('list_tags', 'Tags', + 'ResourceArn', 'arn') + + @property + def arn(self): + return self.data.get('Arn') From a850f738cfcb5653ed47d6a3ec245209db124f8d Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Tue, 23 Oct 2018 10:47:28 +0200 Subject: [PATCH 44/62] Fix the ARN with specific convention --- skew/resources/aws/apigateway.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/skew/resources/aws/apigateway.py b/skew/resources/aws/apigateway.py index 0601868..2a61d9d 100644 --- a/skew/resources/aws/apigateway.py +++ b/skew/resources/aws/apigateway.py @@ -36,3 +36,10 @@ def filter(cls, arn, resource_id, data): api_id = data.get(cls.Meta.id) LOG.debug('%s == %s', resource_id, api_id) return resource_id == api_id + + @property + def arn(self): + # arn:aws:apigateway:us-east-1::/restapis/vwxyz12345 + return 'arn:aws:apigateway:%s::/restapis/%s' % ( + self._client.region_name, + self.id) From 944df83e6d747fffc6fb8efdfb44b3985c2865b2 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Tue, 23 Oct 2018 11:44:18 +0200 Subject: [PATCH 45/62] Adjust logging --- skew/resources/resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skew/resources/resource.py b/skew/resources/resource.py index b3b9c3c..d4af8c2 100644 --- a/skew/resources/resource.py +++ b/skew/resources/resource.py @@ -75,7 +75,7 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): client = skew.awsclient.get_awsclient( 'resourcegroupstaggingapi', region, account, **kwargs) r = client.call('tag_resources', ResourceARNList=[arn], Tags=tags) - logging.warn('Tag ARN %s : %s', arn, r) + LOG.debug('Tag ARN %s, r=%s', arn, r) def sleek(self): """ From 97d38147ec3c1e3bb832f74e9cfaffcdff8fc437 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Tue, 23 Oct 2018 11:44:30 +0200 Subject: [PATCH 46/62] Tags using OpsWorks API instead of ResourceGroupsTaggingAPI --- skew/resources/aws/opsworks.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/skew/resources/aws/opsworks.py b/skew/resources/aws/opsworks.py index 60b4b79..59cec74 100644 --- a/skew/resources/aws/opsworks.py +++ b/skew/resources/aws/opsworks.py @@ -15,6 +15,7 @@ import logging from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient LOG = logging.getLogger(__name__) @@ -25,7 +26,7 @@ class Stack(AWSResource): class Meta(object): service = 'opsworks' type = 'stack' - resourcegroups_tagging = True # only support regional, not classic (us-east-1) + resourcegroups_tagging = False enum_spec = ('describe_stacks', 'Stacks', None) detail_spec = None id = 'StackId' @@ -39,3 +40,12 @@ class Meta(object): @property def arn(self): return self.data.get('Arn') + + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + # ResourceGroupsTaggingAPI supports regional stacks, but not classic (us-east-1) + # opsworks.tag_resource() supports both + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + r = client.call('tag_resource', ResourceArn=arn, Tags=tags) + LOG.debug('Tag ARN %s, r=%s', arn, r) From 9a55a06fa96ce6b20f536789d095d3266235032a Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Tue, 23 Oct 2018 16:18:12 +0200 Subject: [PATCH 47/62] Fix bucket Name --- skew/resources/aws/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skew/resources/aws/s3.py b/skew/resources/aws/s3.py index 0b22922..e174419 100644 --- a/skew/resources/aws/s3.py +++ b/skew/resources/aws/s3.py @@ -54,7 +54,7 @@ class Meta(object): detail_spec = ('list_objects', 'Bucket', 'Contents[]') id = 'Name' filter_name = None - name = 'BucketName' + name = 'Name' date = 'CreationDate' dimension = None tags_spec = ('get_bucket_tagging', 'TagSet[]', From 6f5207b44ec90a2e64779e941fcef626b5261436 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Fri, 16 Nov 2018 10:40:29 +0100 Subject: [PATCH 48/62] Set EC2 Instance name from its tag Name (or InstanceId) --- skew/resources/aws/ec2.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/skew/resources/aws/ec2.py b/skew/resources/aws/ec2.py index 3756208..b78b0e0 100644 --- a/skew/resources/aws/ec2.py +++ b/skew/resources/aws/ec2.py @@ -34,6 +34,10 @@ class Meta(object): def parent(self): return self.data['ImageId'] + def __init__(self, client, data, query=None): + super(Instance, self).__init__(client, data, query) + # Asset name is get by tags if defined, or is InstanceId + self._name = self.tags.get('Name', self.data['InstanceId']) class SecurityGroup(AWSResource): From b5864509aa55d57173fac834dac894f74ff7d491 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Wed, 21 Nov 2018 10:59:26 +0100 Subject: [PATCH 49/62] Fix sleek, set at DBInstance instead of DBSecurityGroup --- skew/resources/aws/rds.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/skew/resources/aws/rds.py b/skew/resources/aws/rds.py index b6bcf96..ebe1fc6 100644 --- a/skew/resources/aws/rds.py +++ b/skew/resources/aws/rds.py @@ -39,6 +39,9 @@ def arn(self): self._client.region_name, self._client.account_id, self.resourcetype, self.id) + def sleek(self): + self.data['LatestRestorableTime'] = '' + class DBSecurityGroup(AWSResource): @@ -61,6 +64,3 @@ def arn(self): self._client.service_name, self._client.region_name, self._client.account_id, self.resourcetype, self.id) - - def sleek(self): - self.data['LatestRestorableTime'] = '' From 8a75751a4200208e345fbd4fba27f1e757a666c1 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Thu, 29 Nov 2018 09:35:54 +0100 Subject: [PATCH 50/62] Implement in sleek method the lists consitant ordering --- skew/resources/aws/autoscaling.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/skew/resources/aws/autoscaling.py b/skew/resources/aws/autoscaling.py index 31d4385..1a1b8ca 100644 --- a/skew/resources/aws/autoscaling.py +++ b/skew/resources/aws/autoscaling.py @@ -34,8 +34,6 @@ class Meta(object): filter_type = 'list' def __init__(self, client, data, query=None): - # Always save the list in the same order to avoid false changes detection - data['EnabledMetrics'].sort(key=lambda item:item['Metric']) super(AutoScalingGroup, self).__init__(client, data, query) self._arn_query = jmespath.compile('AutoScalingGroupARN') @@ -43,6 +41,11 @@ def __init__(self, client, data, query=None): def arn(self): return self._arn_query.search(self.data) + def sleek(self): + # Always render lists in the same order to avoid false changes detection + self.data['EnabledMetrics'].sort(key=lambda item: item['Metric']) + self.data['SuspendedProcesses'].sort(key=str) + @classmethod def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): client = get_awsclient( From 57606de7196bbff111d142b8e487a78ef53ee1d1 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Wed, 26 Dec 2018 09:18:48 +0100 Subject: [PATCH 51/62] Compatibility fix for ECS, that use lowercase 'key' and 'value' as dict keys --- skew/resources/aws/__init__.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/skew/resources/aws/__init__.py b/skew/resources/aws/__init__.py index 28adb8a..a2eadc2 100644 --- a/skew/resources/aws/__init__.py +++ b/skew/resources/aws/__init__.py @@ -170,12 +170,16 @@ def tags(self): _tags = self.data['Tags'] if isinstance(_tags, list): for kvpair in _tags: - if kvpair['Key'] in self._tags: - if not isinstance(self._tags[kvpair['Key']], list): - self._tags[kvpair['Key']] = [self._tags[kvpair['Key']]] - self._tags[kvpair['Key']].append(kvpair['Value']) + # Compatibility fix for ECS, that use lowercase 'key' and 'value' as dict keys + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ecs.html#ECS.Client.list_tags_for_resource + tags_key = kvpair.get('Key', kvpair.get('key')) + tags_value = kvpair.get('Value', kvpair.get('value')) + if tags_key in self._tags: + if not isinstance(self._tags[tags_key], list): + self._tags[tags_key] = [self._tags[tags_key]] + self._tags[tags_key].append(tags_value) else: - self._tags[kvpair['Key']] = kvpair['Value'] + self._tags[tags_key] = tags_value elif isinstance(_tags, dict): self._tags = _tags return self._tags From bdc5b0f03e9f8eb30e860b05e2e1e28a0fd80671 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Wed, 26 Dec 2018 09:20:49 +0100 Subject: [PATCH 52/62] Add tagging support for ECS --- skew/resources/aws/ecs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/skew/resources/aws/ecs.py b/skew/resources/aws/ecs.py index 7927755..046137d 100644 --- a/skew/resources/aws/ecs.py +++ b/skew/resources/aws/ecs.py @@ -31,7 +31,8 @@ class Meta(object): enum_spec = ('list_clusters', 'clusterArns', None) detail_spec = ('describe_clusters', 'clusters', 'clusters[0]') id = None - + tags_spec = ('list_tags_for_resource', 'tags[]', + 'resourceArn', 'arn') filter_name = None name = 'clusterName' From 3ef8b1a7aef09f817e29bdfa0313efd1a89716f9 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Wed, 26 Dec 2018 09:21:14 +0100 Subject: [PATCH 53/62] New method unset_tags() --- skew/resources/aws/autoscaling.py | 11 +++++++++++ skew/resources/aws/cloudfront.py | 6 ++++++ skew/resources/aws/efs.py | 8 ++++++++ skew/resources/aws/opsworks.py | 9 +++++++++ skew/resources/resource.py | 8 ++++++++ 5 files changed, 42 insertions(+) diff --git a/skew/resources/aws/autoscaling.py b/skew/resources/aws/autoscaling.py index 1a1b8ca..0893525 100644 --- a/skew/resources/aws/autoscaling.py +++ b/skew/resources/aws/autoscaling.py @@ -57,6 +57,17 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): tags_list = [dict(Key=k, Value=str(v), **addon) for k, v in tags.items()] return client.call('create_or_update_tags', Tags=tags_list) + @classmethod + def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + asg_name = arn.split(':')[7].split('/')[1] + addon = dict(ResourceId=asg_name, + ResourceType='auto-scaling-group', + PropagateAtLaunch=False) + tags_list = [dict(Key=k, Value=str(''), **addon) for k in tags_keys] + return client.call('delete_tags', Tags=tags_list) + class LaunchConfiguration(AWSResource): diff --git a/skew/resources/aws/cloudfront.py b/skew/resources/aws/cloudfront.py index 61f4793..6e9b050 100644 --- a/skew/resources/aws/cloudfront.py +++ b/skew/resources/aws/cloudfront.py @@ -43,3 +43,9 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): cls.Meta.service, region, account, **kwargs) tags_list = [dict(Key=k, Value=str(v)) for k, v in tags.items()] return client.call('tag_resource', Resource=arn, Tags=dict(Items=tags_list)) + + @classmethod + def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + return client.call('untag_resource', Resource=arn, TagsKeys=dict(Items=tags_keys)) diff --git a/skew/resources/aws/efs.py b/skew/resources/aws/efs.py index 58e9ac0..f706132 100644 --- a/skew/resources/aws/efs.py +++ b/skew/resources/aws/efs.py @@ -59,3 +59,11 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): x = client.call('create_tags', FileSystemId=arn.split('/')[-1], Tags=tags_list) print(x, arn.split('/')[-1], tags_list) return x + + @classmethod + def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + x = client.call('delete_tags', FileSystemId=arn.split('/')[-1], TagsKeys=tags_keys) + print(x, arn.split('/')[-1], tags_keys) + return x diff --git a/skew/resources/aws/opsworks.py b/skew/resources/aws/opsworks.py index 59cec74..08441ae 100644 --- a/skew/resources/aws/opsworks.py +++ b/skew/resources/aws/opsworks.py @@ -49,3 +49,12 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): cls.Meta.service, region, account, **kwargs) r = client.call('tag_resource', ResourceArn=arn, Tags=tags) LOG.debug('Tag ARN %s, r=%s', arn, r) + + @classmethod + def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): + # ResourceGroupsTaggingAPI supports regional stacks, but not classic (us-east-1) + # opsworks.untag_resource() supports both + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + r = client.call('untag_resource', ResourceArn=arn, TagsKeys=tags_keys) + LOG.debug('UnTag ARN %s, r=%s', arn, r) diff --git a/skew/resources/resource.py b/skew/resources/resource.py index d4af8c2..7eb57ea 100644 --- a/skew/resources/resource.py +++ b/skew/resources/resource.py @@ -77,6 +77,14 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): r = client.call('tag_resources', ResourceARNList=[arn], Tags=tags) LOG.debug('Tag ARN %s, r=%s', arn, r) + @classmethod + def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): + if hasattr(cls.Meta, 'resourcegroups_tagging') and (cls.Meta.resourcegroups_tagging): + client = skew.awsclient.get_awsclient( + 'resourcegroupstaggingapi', region, account, **kwargs) + r = client.call('untag_resources', ResourceARNList=[arn], TagsKeys=tags_keys) + LOG.debug('UnTag ARN %s, r=%s', arn, r) + def sleek(self): """ overrides datas frequently varied by a static value. From 982a961247259338ce3cfbb36910e83ae5ca60b7 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Wed, 26 Dec 2018 15:03:37 +0100 Subject: [PATCH 54/62] Typo in TagKeys --- skew/resources/aws/autoscaling.py | 2 +- skew/resources/aws/cloudfront.py | 2 +- skew/resources/aws/efs.py | 2 +- skew/resources/aws/opsworks.py | 2 +- skew/resources/resource.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/skew/resources/aws/autoscaling.py b/skew/resources/aws/autoscaling.py index 0893525..199f30e 100644 --- a/skew/resources/aws/autoscaling.py +++ b/skew/resources/aws/autoscaling.py @@ -65,7 +65,7 @@ def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs) addon = dict(ResourceId=asg_name, ResourceType='auto-scaling-group', PropagateAtLaunch=False) - tags_list = [dict(Key=k, Value=str(''), **addon) for k in tags_keys] + tags_list = [dict(Key=k, **addon) for k in tags_keys] return client.call('delete_tags', Tags=tags_list) diff --git a/skew/resources/aws/cloudfront.py b/skew/resources/aws/cloudfront.py index 6e9b050..f05fc7a 100644 --- a/skew/resources/aws/cloudfront.py +++ b/skew/resources/aws/cloudfront.py @@ -48,4 +48,4 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): client = get_awsclient( cls.Meta.service, region, account, **kwargs) - return client.call('untag_resource', Resource=arn, TagsKeys=dict(Items=tags_keys)) + return client.call('untag_resource', Resource=arn, TagKeys=dict(Items=tags_keys)) diff --git a/skew/resources/aws/efs.py b/skew/resources/aws/efs.py index f706132..54f2746 100644 --- a/skew/resources/aws/efs.py +++ b/skew/resources/aws/efs.py @@ -64,6 +64,6 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): client = get_awsclient( cls.Meta.service, region, account, **kwargs) - x = client.call('delete_tags', FileSystemId=arn.split('/')[-1], TagsKeys=tags_keys) + x = client.call('delete_tags', FileSystemId=arn.split('/')[-1], TagKeys=tags_keys) print(x, arn.split('/')[-1], tags_keys) return x diff --git a/skew/resources/aws/opsworks.py b/skew/resources/aws/opsworks.py index 08441ae..6997c0e 100644 --- a/skew/resources/aws/opsworks.py +++ b/skew/resources/aws/opsworks.py @@ -56,5 +56,5 @@ def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs) # opsworks.untag_resource() supports both client = get_awsclient( cls.Meta.service, region, account, **kwargs) - r = client.call('untag_resource', ResourceArn=arn, TagsKeys=tags_keys) + r = client.call('untag_resource', ResourceArn=arn, TagKeys=tags_keys) LOG.debug('UnTag ARN %s, r=%s', arn, r) diff --git a/skew/resources/resource.py b/skew/resources/resource.py index 7eb57ea..bd4c35d 100644 --- a/skew/resources/resource.py +++ b/skew/resources/resource.py @@ -82,7 +82,7 @@ def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs) if hasattr(cls.Meta, 'resourcegroups_tagging') and (cls.Meta.resourcegroups_tagging): client = skew.awsclient.get_awsclient( 'resourcegroupstaggingapi', region, account, **kwargs) - r = client.call('untag_resources', ResourceARNList=[arn], TagsKeys=tags_keys) + r = client.call('untag_resources', ResourceARNList=[arn], TagKeys=tags_keys) LOG.debug('UnTag ARN %s, r=%s', arn, r) def sleek(self): From 17a31d3b68099ee44afe8c56fd54b028df07d7fe Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Fri, 28 Dec 2018 11:24:32 +0100 Subject: [PATCH 55/62] Cosmetic change, replace 'tags_keys' by 'tag_keys' for wording consistency --- skew/resources/aws/autoscaling.py | 4 ++-- skew/resources/aws/cloudfront.py | 4 ++-- skew/resources/aws/efs.py | 6 ++---- skew/resources/aws/opsworks.py | 4 ++-- skew/resources/resource.py | 4 ++-- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/skew/resources/aws/autoscaling.py b/skew/resources/aws/autoscaling.py index 199f30e..2eb880b 100644 --- a/skew/resources/aws/autoscaling.py +++ b/skew/resources/aws/autoscaling.py @@ -58,14 +58,14 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): return client.call('create_or_update_tags', Tags=tags_list) @classmethod - def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): + def unset_tags(cls, arn, region, account, tag_keys, resource_id=None, **kwargs): client = get_awsclient( cls.Meta.service, region, account, **kwargs) asg_name = arn.split(':')[7].split('/')[1] addon = dict(ResourceId=asg_name, ResourceType='auto-scaling-group', PropagateAtLaunch=False) - tags_list = [dict(Key=k, **addon) for k in tags_keys] + tags_list = [dict(Key=k, **addon) for k in tag_keys] return client.call('delete_tags', Tags=tags_list) diff --git a/skew/resources/aws/cloudfront.py b/skew/resources/aws/cloudfront.py index f05fc7a..c54010a 100644 --- a/skew/resources/aws/cloudfront.py +++ b/skew/resources/aws/cloudfront.py @@ -45,7 +45,7 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): return client.call('tag_resource', Resource=arn, Tags=dict(Items=tags_list)) @classmethod - def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): + def unset_tags(cls, arn, region, account, tag_keys, resource_id=None, **kwargs): client = get_awsclient( cls.Meta.service, region, account, **kwargs) - return client.call('untag_resource', Resource=arn, TagKeys=dict(Items=tags_keys)) + return client.call('untag_resource', Resource=arn, TagKeys=dict(Items=tag_keys)) diff --git a/skew/resources/aws/efs.py b/skew/resources/aws/efs.py index 54f2746..d651ddf 100644 --- a/skew/resources/aws/efs.py +++ b/skew/resources/aws/efs.py @@ -57,13 +57,11 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): cls.Meta.service, region, account, **kwargs) tags_list = [dict(Key=k, Value=str(v)) for k, v in tags.items()] x = client.call('create_tags', FileSystemId=arn.split('/')[-1], Tags=tags_list) - print(x, arn.split('/')[-1], tags_list) return x @classmethod - def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): + def unset_tags(cls, arn, region, account, tag_keys, resource_id=None, **kwargs): client = get_awsclient( cls.Meta.service, region, account, **kwargs) - x = client.call('delete_tags', FileSystemId=arn.split('/')[-1], TagKeys=tags_keys) - print(x, arn.split('/')[-1], tags_keys) + x = client.call('delete_tags', FileSystemId=arn.split('/')[-1], TagKeys=tag_keys) return x diff --git a/skew/resources/aws/opsworks.py b/skew/resources/aws/opsworks.py index 6997c0e..ee6fa6a 100644 --- a/skew/resources/aws/opsworks.py +++ b/skew/resources/aws/opsworks.py @@ -51,10 +51,10 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): LOG.debug('Tag ARN %s, r=%s', arn, r) @classmethod - def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): + def unset_tags(cls, arn, region, account, tag_keys, resource_id=None, **kwargs): # ResourceGroupsTaggingAPI supports regional stacks, but not classic (us-east-1) # opsworks.untag_resource() supports both client = get_awsclient( cls.Meta.service, region, account, **kwargs) - r = client.call('untag_resource', ResourceArn=arn, TagKeys=tags_keys) + r = client.call('untag_resource', ResourceArn=arn, TagKeys=tag_keys) LOG.debug('UnTag ARN %s, r=%s', arn, r) diff --git a/skew/resources/resource.py b/skew/resources/resource.py index bd4c35d..a6c6f35 100644 --- a/skew/resources/resource.py +++ b/skew/resources/resource.py @@ -78,11 +78,11 @@ def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): LOG.debug('Tag ARN %s, r=%s', arn, r) @classmethod - def unset_tags(cls, arn, region, account, tags_keys, resource_id=None, **kwargs): + def unset_tags(cls, arn, region, account, tag_keys, resource_id=None, **kwargs): if hasattr(cls.Meta, 'resourcegroups_tagging') and (cls.Meta.resourcegroups_tagging): client = skew.awsclient.get_awsclient( 'resourcegroupstaggingapi', region, account, **kwargs) - r = client.call('untag_resources', ResourceARNList=[arn], TagKeys=tags_keys) + r = client.call('untag_resources', ResourceARNList=[arn], TagKeys=tag_keys) LOG.debug('UnTag ARN %s, r=%s', arn, r) def sleek(self): From 4f0ec5d4c6ff45fbc41b38adde4a8e82714552df Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Fri, 28 Dec 2018 15:16:54 +0100 Subject: [PATCH 56/62] Improve acm resource add a filter method add details fetch --- skew/resources/aws/acm.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/skew/resources/aws/acm.py b/skew/resources/aws/acm.py index 8402651..147d295 100644 --- a/skew/resources/aws/acm.py +++ b/skew/resources/aws/acm.py @@ -17,7 +17,6 @@ import jmespath from skew.resources.aws import AWSResource -from skew.awsclient import get_awsclient LOG = logging.getLogger(__name__) @@ -30,15 +29,32 @@ class Meta(object): type = 'certificate' resourcegroups_tagging = True enum_spec = ('list_certificates', 'CertificateSummaryList', None) - detail_spec = None + detail_spec = ('describe_certificate', 'CertificateArn', 'Certificate') id = 'CertificateArn' tags_spec = ('list_tags_for_certificate', 'Tags[]', 'CertificateArn', 'id') filter_name = None name = 'DomainName' - date = None + date = 'CreatedAt' dimension = None + @classmethod + def filter(cls, arn, resource_id, data): + certificate_id = data.get(cls.Meta.id).split('/')[-1] + LOG.debug('%s == %s', resource_id, certificate_id) + return resource_id == certificate_id + @property def arn(self): return self.data['CertificateArn'] + + def __init__(self, client, data, query=None): + super(Certificate, self).__init__(client, data, query) + + self._id = data['CertificateArn'] + + detail_op, param_name, detail_path = self.Meta.detail_spec + params = {param_name: data['CertificateArn']} + data = client.call(detail_op, **params) + + self.data = jmespath.search(detail_path, data) From 4e57e79c23767dfbc2505b235d0f361602a8a1c6 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Tue, 8 Jan 2019 17:57:04 +0100 Subject: [PATCH 57/62] Disable tag support while Lambda boto3 version not yet support ecs/list_tags_for_resource --- skew/resources/aws/ecs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/skew/resources/aws/ecs.py b/skew/resources/aws/ecs.py index 046137d..b539ee9 100644 --- a/skew/resources/aws/ecs.py +++ b/skew/resources/aws/ecs.py @@ -31,8 +31,9 @@ class Meta(object): enum_spec = ('list_clusters', 'clusterArns', None) detail_spec = ('describe_clusters', 'clusters', 'clusters[0]') id = None - tags_spec = ('list_tags_for_resource', 'tags[]', - 'resourceArn', 'arn') + tags_spec = None # Not yet supported on Lambda boto3 included + # tags_spec = ('list_tags_for_resource', 'tags[]', + # 'resourceArn', 'arn') filter_name = None name = 'clusterName' From bbd55ace60e09fd21bf586cdc2bed8d970e7c873 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Tue, 15 Jan 2019 17:02:34 +0100 Subject: [PATCH 58/62] Asset name is get by tags if defined, or is FileSystemId --- skew/resources/aws/efs.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/skew/resources/aws/efs.py b/skew/resources/aws/efs.py index d651ddf..85429e3 100644 --- a/skew/resources/aws/efs.py +++ b/skew/resources/aws/efs.py @@ -48,6 +48,11 @@ def arn(self): self._client.account_id, 'file-system-id', self.id) + def __init__(self, client, data, query=None): + super(Filesystem, self).__init__(client, data, query) + # Asset name is get by tags if defined, or is FileSystemId + self._name = self.tags.get('Name', self.data['FileSystemId']) + def sleek(self): self.data['SizeInBytes'] = 0 From 9dda93738d4bef857b31eb0b515e8ea0d2a84f0a Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Tue, 15 Jan 2019 17:02:54 +0100 Subject: [PATCH 59/62] Attribute DisplayName is not relevant, set Id as name instead --- skew/resources/aws/sns.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/skew/resources/aws/sns.py b/skew/resources/aws/sns.py index 216e6d6..9c772ab 100644 --- a/skew/resources/aws/sns.py +++ b/skew/resources/aws/sns.py @@ -54,6 +54,9 @@ def __init__(self, client, data, query=None): self.data = jmespath.search(detail_path, data) + # Attribute DisplayName is not relevant, set Id as name instead + self._name = self._id + class Subscription(AWSResource): From b2e667222245a49c4d0e7a01b6791557bbbee6ae Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Thu, 17 Jan 2019 15:15:07 +0100 Subject: [PATCH 60/62] Add Cloudsearch Domain --- skew/resources/__init__.py | 1 + skew/resources/aws/cloudsearch.py | 42 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 skew/resources/aws/cloudsearch.py diff --git a/skew/resources/__init__.py b/skew/resources/__init__.py index 5dd233e..ec57ab2 100644 --- a/skew/resources/__init__.py +++ b/skew/resources/__init__.py @@ -23,6 +23,7 @@ 'aws.autoscaling.launchConfigurationName': 'aws.autoscaling.LaunchConfiguration', 'aws.cloudfront.distribution': 'aws.cloudfront.Distribution', 'aws.cloudformation.stack': 'aws.cloudformation.Stack', + 'aws.cloudsearch.domain': 'aws.cloudsearch.Domain', 'aws.cloudwatch.alarm': 'aws.cloudwatch.Alarm', 'aws.dynamodb.table': 'aws.dynamodb.Table', 'aws.ec2.address': 'aws.ec2.Address', diff --git a/skew/resources/aws/cloudsearch.py b/skew/resources/aws/cloudsearch.py new file mode 100644 index 0000000..bb250cb --- /dev/null +++ b/skew/resources/aws/cloudsearch.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Domain(AWSResource): + + class Meta(object): + service = 'cloudsearch' + type = 'domain' + resourcegroups_tagging = False + enum_spec = ('describe_domains', 'DomainStatusList', None) + detail_spec = None + id = 'ARN' + tags_spec = None + filter_name = None + name = 'DomainName' + date = 'Created' + dimension = None + + @property + def arn(self): + return self.data['ARN'] From abecda847a6013e6afd8393eb1b1814406006ce8 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Tue, 22 Jan 2019 09:24:25 +0100 Subject: [PATCH 61/62] Fix bucket no detected when LocationConstraint == 'EU' --- skew/resources/aws/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skew/resources/aws/s3.py b/skew/resources/aws/s3.py index e174419..315f152 100644 --- a/skew/resources/aws/s3.py +++ b/skew/resources/aws/s3.py @@ -39,7 +39,7 @@ def enumerate(cls, arn, region, account, resource_id=None, **kwargs): location = response.get('LocationConstraint', 'us-east-1') if location is None: location = 'us-east-1' - if location is 'EU': + if location == 'EU': location = 'eu-west-1' cls._location_cache[r.id] = location if location == region: From 817f34c01e95237a0f5905ed5987d39fcdcbd688 Mon Sep 17 00:00:00 2001 From: Christophe Morio Date: Tue, 22 Jan 2019 15:22:06 +0100 Subject: [PATCH 62/62] Use native SQS tagging action --- skew/resources/aws/sqs.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/skew/resources/aws/sqs.py b/skew/resources/aws/sqs.py index 4f6e5af..4af732b 100644 --- a/skew/resources/aws/sqs.py +++ b/skew/resources/aws/sqs.py @@ -13,6 +13,7 @@ # language governing permissions and limitations under the License. from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient class Queue(AWSResource): @@ -20,7 +21,7 @@ class Queue(AWSResource): class Meta(object): service = 'sqs' type = 'queue' - resourcegroups_tagging = True + resourcegroups_tagging = False enum_spec = ('list_queues', 'QueueUrls', None) detail_spec = ('get_queue_attributes', 'QueueUrl', 'QueueUrl') id = 'QueueUrl' @@ -29,15 +30,43 @@ class Meta(object): name = 'QueueName' date = None dimension = 'QueueName' + tags_spec = None def __init__(self, client, data, query=None): super(Queue, self).__init__(client, data, query) self.data = {self.Meta.id: data, 'QueueName': data.split('/')[-1]} self._id = self.data['QueueName'] + response = client.call('list_queue_tags', + QueueUrl=self.data['QueueUrl']) + self._tags = response.get('Tags', {}) @property def arn(self): return 'arn:aws:sqs:%s:%s:%s' % ( self._client.region_name, - self._client.account_id, self.id) \ No newline at end of file + self._client.account_id, self.id) + + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + queue_name = arn.split(':')[5] + queue_url = 'https://sqs.{}.amazonaws.com/{}/{}'.format(region, + account, + queue_name) + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + return client.call('tag_queue', + QueueUrl=queue_url, + Tags=tags) + + @classmethod + def unset_tags(cls, arn, region, account, tag_keys, resource_id=None, **kwargs): + queue_name = arn.split(':')[5] + queue_url = 'https://sqs.{}.amazonaws.com/{}/{}'.format(region, + account, + queue_name) + client = get_awsclient( + cls.Meta.service, region, account, **kwargs) + return client.call('untag_queue', + QueueUrl=queue_url, + TagKeys=tag_keys)