From 764bb69b62f5401752489355a39c062688ff813f Mon Sep 17 00:00:00 2001 From: Derek Waters Date: Thu, 29 Aug 2024 12:45:03 -0400 Subject: [PATCH 01/67] Add a credential plugin to implement AWS AssumeRole functionality --- src/awx_plugins/credentials/aws_assumerole.py | 104 ++++++++++++++++++ tests/credential_plugins_test.py | 57 ++++++++++ 2 files changed, 161 insertions(+) create mode 100644 src/awx_plugins/credentials/aws_assumerole.py diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py new file mode 100644 index 0000000000..7117f6c445 --- /dev/null +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -0,0 +1,104 @@ +import boto3 +import hashlib +import datetime + +from .plugin import CredentialPlugin +from django.utils.translation import gettext_lazy as _ + +try: + from botocore.exceptions import ClientError +except ImportError: + pass # caught by AnsibleAWSModule + +_aws_cred_cache = {} + + +assume_role_inputs = { + 'fields': [ + { + 'id': 'access_key', + 'label': _('AWS Access Key'), + 'type': 'string', + 'secret': True, + 'help_text': _('The optional AWS access key for the user who will assume the role'), + }, + { + 'id': 'secret_key', + 'label': 'AWS Secret Key', + 'type': 'string', + 'secret': True, + 'help_text': _('The optional AWS secret key for the user who will assume the role'), + }, + { + 'id': 'external_id', + 'label': 'External ID', + 'type': 'string', + 'help_text': _('The optional External ID which will be provided to the assume role API'), + }, + {'id': 'role_arn', 'label': 'AWS ARN Role Name', 'type': 'string', 'secret': True, 'help_text': _('The ARN Role Name to be assumed in AWS')}, + ], + 'metadata': [ + { + 'id': 'identifier', + 'label': 'Identifier', + 'type': 'string', + 'help_text': _('The name of the key in the assumed AWS role to fetch [AccessKeyId | SecretAccessKey | SessionToken].'), + }, + ], + 'required': ['role_arn'], +} + + +def aws_assumerole_getcreds(access_key, secret_key, role_arn, external_id): + if (access_key is None or len(access_key) == 0) and (secret_key is None or len(secret_key) == 0): + # Connect using credentials in the EE + connection = boto3.client(service_name="sts") + else: + # Connect to AWS using provided credentials + connection = boto3.client(service_name="sts", aws_access_key_id=access_key, aws_secret_access_key=secret_key) + try: + response = connection.assume_role(RoleArn=role_arn, RoleSessionName='AAP_AWS_Role_Session1', ExternalId=external_id) + except ClientError as ce: + raise ValueError(f'Got a bad client response from AWS: {ce.msg}.') + + credentials = response.get("Credentials", {}) + + return credentials + + +def aws_assumerole_backend(**kwargs): + """This backend function actually contacts AWS to assume a given role for the specified user""" + access_key = kwargs.get('access_key') + secret_key = kwargs.get('secret_key') + role_arn = kwargs.get('role_arn') + external_id = kwargs.get('external_id') + identifier = kwargs.get('identifier') + + # Generate a unique SHA256 hash for combo of user access key and ARN + # This should allow two users requesting the same ARN role to have + # separate credentials, and should allow the same user to request + # multiple roles. + # + credential_key_hash = hashlib.sha256((str(access_key or '') + role_arn).encode('utf-8')) + credential_key = credential_key_hash.hexdigest() + + credentials = _aws_cred_cache.get(credential_key, None) + + # If there are no credentials for this user/ARN *or* the credentials + # we have in the cache have expired, then we need to contact AWS again. + # + if (credentials is None) or (credentials['Expiration'] < datetime.datetime.now(credentials['Expiration'].tzinfo)): + + credentials = aws_assumerole_getcreds(access_key, secret_key, role_arn, external_id) + + _aws_cred_cache[credential_key] = credentials + + credentials = _aws_cred_cache.get(credential_key, None) + + if identifier in credentials: + return credentials[identifier] + + raise ValueError(f'Could not find a value for {identifier}.') + + +aws_assumerole_plugin = CredentialPlugin('AWS Assume Role Plugin', inputs=assume_role_inputs, backend=aws_assumerole_backend) \ No newline at end of file diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index 2c2f3eae67..3754206f8c 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -3,9 +3,11 @@ from unittest import mock +import datetime import pytest from awx_plugins.credentials import hashivault +from awx_plugins.credentials import aws_assumerole def test_imported_azure_cloud_sdk_vars() -> None: @@ -133,6 +135,61 @@ def test_hashivault_handle_auth_not_enough_args() -> None: with pytest.raises(Exception): hashivault.handle_auth() +def test_aws_assumerole_with_accesssecret(): + kwargs = { + 'access_key': 'my_access_key', + 'secret_key': 'my_secret_key', + 'role_arn': 'the_arn', + 'identifier': 'access_token', + } + with mock.patch.object(aws_assumerole, 'aws_assumerole_getcreds') as method_mock: + method_mock.return_value = { + 'access_key': 'the_access_key', + 'secret_key': 'the_secret_key', + 'access_token': 'the_access_token', + 'Expiration': datetime.datetime.today() + datetime.timedelta(days=1), + } + token = aws_assumerole.aws_assumerole_backend(**kwargs) + method_mock.assert_called_with(kwargs.get('access_key'), kwargs.get('secret_key'), kwargs.get('role_arn'), None) + assert token == 'the_access_token' + kwargs['identifier'] = 'secret_key' + method_mock.reset_mock() + token = aws_assumerole.aws_assumerole_backend(**kwargs) + method_mock.assert_not_called() + assert token == 'the_secret_key' + kwargs['identifier'] = 'access_key' + method_mock.reset_mock() + token = aws_assumerole.aws_assumerole_backend(**kwargs) + method_mock.assert_not_called() + assert token == 'the_access_key' + + +def test_aws_assumerole_with_arnonly(): + kwargs = { + 'role_arn': 'the_arn', + 'identifier': 'access_token', + } + with mock.patch.object(aws_assumerole, 'aws_assumerole_getcreds') as method_mock: + method_mock.return_value = { + 'access_key': 'the_access_key', + 'secret_key': 'the_secret_key', + 'access_token': 'the_access_token', + 'Expiration': datetime.datetime.today() + datetime.timedelta(days=1), + } + token = aws_assumerole.aws_assumerole_backend(**kwargs) + method_mock.assert_called_with(None, None, kwargs.get('role_arn'), None) + assert token == 'the_access_token' + kwargs['identifier'] = 'secret_key' + method_mock.reset_mock() + token = aws_assumerole.aws_assumerole_backend(**kwargs) + method_mock.assert_not_called() + assert token == 'the_secret_key' + kwargs['identifier'] = 'access_key' + method_mock.reset_mock() + token = aws_assumerole.aws_assumerole_backend(**kwargs) + method_mock.assert_not_called() + assert token == 'the_access_key' + class TestDelineaImports: """These module have a try-except for ImportError which will allow using From 5d19ff16fa467b2bb925a175a7f841658b3e9b3f Mon Sep 17 00:00:00 2001 From: Derek Waters Date: Thu, 29 Aug 2024 12:48:49 -0400 Subject: [PATCH 02/67] Fix EoF --- src/awx_plugins/credentials/aws_assumerole.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index 7117f6c445..cba2a54434 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -101,4 +101,4 @@ def aws_assumerole_backend(**kwargs): raise ValueError(f'Could not find a value for {identifier}.') -aws_assumerole_plugin = CredentialPlugin('AWS Assume Role Plugin', inputs=assume_role_inputs, backend=aws_assumerole_backend) \ No newline at end of file +aws_assumerole_plugin = CredentialPlugin('AWS Assume Role Plugin', inputs=assume_role_inputs, backend=aws_assumerole_backend) From cf6d48140d70a337cfea62ae5e39b9c18d50f692 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 16:52:02 +0000 Subject: [PATCH 03/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/awx_plugins/credentials/aws_assumerole.py | 105 ++++++++++-------- tests/credential_plugins_test.py | 15 ++- 2 files changed, 69 insertions(+), 51 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index cba2a54434..bc74ae7af5 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -1,10 +1,13 @@ -import boto3 -import hashlib import datetime +import hashlib -from .plugin import CredentialPlugin from django.utils.translation import gettext_lazy as _ +import boto3 + +from .plugin import CredentialPlugin + + try: from botocore.exceptions import ClientError except ImportError: @@ -13,55 +16,59 @@ _aws_cred_cache = {} -assume_role_inputs = { - 'fields': [ - { - 'id': 'access_key', - 'label': _('AWS Access Key'), - 'type': 'string', - 'secret': True, - 'help_text': _('The optional AWS access key for the user who will assume the role'), - }, - { - 'id': 'secret_key', - 'label': 'AWS Secret Key', - 'type': 'string', - 'secret': True, - 'help_text': _('The optional AWS secret key for the user who will assume the role'), - }, - { - 'id': 'external_id', - 'label': 'External ID', - 'type': 'string', - 'help_text': _('The optional External ID which will be provided to the assume role API'), - }, - {'id': 'role_arn', 'label': 'AWS ARN Role Name', 'type': 'string', 'secret': True, 'help_text': _('The ARN Role Name to be assumed in AWS')}, - ], - 'metadata': [ - { - 'id': 'identifier', - 'label': 'Identifier', - 'type': 'string', - 'help_text': _('The name of the key in the assumed AWS role to fetch [AccessKeyId | SecretAccessKey | SessionToken].'), - }, - ], - 'required': ['role_arn'], -} +assume_role_inputs = {'fields': [{'id': 'access_key', + 'label': _('AWS Access Key'), + 'type': 'string', + 'secret': True, + 'help_text': _('The optional AWS access key for the user who will assume the role'), + }, + {'id': 'secret_key', + 'label': 'AWS Secret Key', + 'type': 'string', + 'secret': True, + 'help_text': _('The optional AWS secret key for the user who will assume the role'), + }, + {'id': 'external_id', + 'label': 'External ID', + 'type': 'string', + 'help_text': _('The optional External ID which will be provided to the assume role API'), + }, + {'id': 'role_arn', + 'label': 'AWS ARN Role Name', + 'type': 'string', + 'secret': True, + 'help_text': _('The ARN Role Name to be assumed in AWS')}, + ], + 'metadata': [{'id': 'identifier', + 'label': 'Identifier', + 'type': 'string', + 'help_text': _('The name of the key in the assumed AWS role to fetch [AccessKeyId | SecretAccessKey | SessionToken].'), + }, + ], + 'required': ['role_arn'], + } def aws_assumerole_getcreds(access_key, secret_key, role_arn, external_id): - if (access_key is None or len(access_key) == 0) and (secret_key is None or len(secret_key) == 0): + if (access_key is None or len(access_key) == 0) and ( + secret_key is None or len(secret_key) == 0): # Connect using credentials in the EE - connection = boto3.client(service_name="sts") + connection = boto3.client(service_name='sts') else: # Connect to AWS using provided credentials - connection = boto3.client(service_name="sts", aws_access_key_id=access_key, aws_secret_access_key=secret_key) + connection = boto3.client( + service_name='sts', + aws_access_key_id=access_key, + aws_secret_access_key=secret_key) try: - response = connection.assume_role(RoleArn=role_arn, RoleSessionName='AAP_AWS_Role_Session1', ExternalId=external_id) + response = connection.assume_role( + RoleArn=role_arn, + RoleSessionName='AAP_AWS_Role_Session1', + ExternalId=external_id) except ClientError as ce: raise ValueError(f'Got a bad client response from AWS: {ce.msg}.') - credentials = response.get("Credentials", {}) + credentials = response.get('Credentials', {}) return credentials @@ -79,7 +86,8 @@ def aws_assumerole_backend(**kwargs): # separate credentials, and should allow the same user to request # multiple roles. # - credential_key_hash = hashlib.sha256((str(access_key or '') + role_arn).encode('utf-8')) + credential_key_hash = hashlib.sha256( + (str(access_key or '') + role_arn).encode('utf-8')) credential_key = credential_key_hash.hexdigest() credentials = _aws_cred_cache.get(credential_key, None) @@ -87,9 +95,11 @@ def aws_assumerole_backend(**kwargs): # If there are no credentials for this user/ARN *or* the credentials # we have in the cache have expired, then we need to contact AWS again. # - if (credentials is None) or (credentials['Expiration'] < datetime.datetime.now(credentials['Expiration'].tzinfo)): + if (credentials is None) or (credentials['Expiration'] < datetime.datetime.now( + credentials['Expiration'].tzinfo)): - credentials = aws_assumerole_getcreds(access_key, secret_key, role_arn, external_id) + credentials = aws_assumerole_getcreds( + access_key, secret_key, role_arn, external_id) _aws_cred_cache[credential_key] = credentials @@ -101,4 +111,7 @@ def aws_assumerole_backend(**kwargs): raise ValueError(f'Could not find a value for {identifier}.') -aws_assumerole_plugin = CredentialPlugin('AWS Assume Role Plugin', inputs=assume_role_inputs, backend=aws_assumerole_backend) +aws_assumerole_plugin = CredentialPlugin( + 'AWS Assume Role Plugin', + inputs=assume_role_inputs, + backend=aws_assumerole_backend) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index 3754206f8c..3d1d6573c8 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -1,13 +1,12 @@ # FIXME: the following violations must be addressed gradually and unignored # mypy: disable-error-code="no-untyped-call" +import datetime from unittest import mock -import datetime import pytest -from awx_plugins.credentials import hashivault -from awx_plugins.credentials import aws_assumerole +from awx_plugins.credentials import aws_assumerole, hashivault def test_imported_azure_cloud_sdk_vars() -> None: @@ -135,6 +134,7 @@ def test_hashivault_handle_auth_not_enough_args() -> None: with pytest.raises(Exception): hashivault.handle_auth() + def test_aws_assumerole_with_accesssecret(): kwargs = { 'access_key': 'my_access_key', @@ -150,7 +150,11 @@ def test_aws_assumerole_with_accesssecret(): 'Expiration': datetime.datetime.today() + datetime.timedelta(days=1), } token = aws_assumerole.aws_assumerole_backend(**kwargs) - method_mock.assert_called_with(kwargs.get('access_key'), kwargs.get('secret_key'), kwargs.get('role_arn'), None) + method_mock.assert_called_with( + kwargs.get('access_key'), + kwargs.get('secret_key'), + kwargs.get('role_arn'), + None) assert token == 'the_access_token' kwargs['identifier'] = 'secret_key' method_mock.reset_mock() @@ -177,7 +181,8 @@ def test_aws_assumerole_with_arnonly(): 'Expiration': datetime.datetime.today() + datetime.timedelta(days=1), } token = aws_assumerole.aws_assumerole_backend(**kwargs) - method_mock.assert_called_with(None, None, kwargs.get('role_arn'), None) + method_mock.assert_called_with( + None, None, kwargs.get('role_arn'), None) assert token == 'the_access_token' kwargs['identifier'] = 'secret_key' method_mock.reset_mock() From 24ce59c12a1027bb1309d6379beaaeca30846f5e Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Thu, 29 Aug 2024 16:16:41 -0400 Subject: [PATCH 04/67] add doc strings and adjust entry point in pyproject toml --- tests/credential_plugins_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index 3d1d6573c8..ec844f0be9 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -136,6 +136,9 @@ def test_hashivault_handle_auth_not_enough_args() -> None: def test_aws_assumerole_with_accesssecret(): + ''' + Test that the aws_assumerole_backend function call returns a token given the access_key and secret_key. + ''' kwargs = { 'access_key': 'my_access_key', 'secret_key': 'my_secret_key', @@ -169,6 +172,9 @@ def test_aws_assumerole_with_accesssecret(): def test_aws_assumerole_with_arnonly(): + ''' + Test backend function with only the role ARN provided. + ''' kwargs = { 'role_arn': 'the_arn', 'identifier': 'access_token', From b50e00314ead1d91280ce87e61d0f6128d3cb4c2 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Thu, 5 Sep 2024 14:05:18 -0400 Subject: [PATCH 05/67] move to monkeypatch for test, add deps and more clean up --- pyproject.toml | 2 + src/awx_plugins/credentials/aws_assumerole.py | 31 +++++-- tests/credential_plugins_test.py | 91 ++++++++++--------- 3 files changed, 69 insertions(+), 55 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bdbdf64e1b..8fa8b8b788 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ # runtime deps # https://packaging.python.org/en/latest/guide "python-dsv-sdk >= 1.0.4", # credentials.thycotic_dsv "python-tss-sdk >= 1.2.1", # credentials.thycotic_tss "requests", # credentials.aim, credentials.centrify_vault, credentials.conjur, credentials.hashivault + "datetime", # credentials.aws_assume_role ] classifiers = [ # Allowlist: https://pypi.org/classifiers/ "Development Status :: 1 - Planning", @@ -83,6 +84,7 @@ centrify_vault_kv = "awx_plugins.credentials.centrify_vault:centrify_plugin" thycotic_dsv = "awx_plugins.credentials.dsv:dsv_plugin" thycotic_tss = "awx_plugins.credentials.tss:tss_plugin" aws_secretsmanager_credential = "awx_plugins.credentials.aws_secretsmanager:aws_secretmanager_plugin" +aws_assume_role = "awx_plugins.credentials.aws_assume_role:aws_assume_role_plugin" [project.entry-points."awx_plugins.inventory"] # new entry points group name azure-rm = "awx_plugins.inventory.plugins:azure_rm" diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index bc74ae7af5..3aa17605af 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -37,7 +37,8 @@ 'label': 'AWS ARN Role Name', 'type': 'string', 'secret': True, - 'help_text': _('The ARN Role Name to be assumed in AWS')}, + 'help_text': _('The ARN Role Name to be assumed in AWS'), + }, ], 'metadata': [{'id': 'identifier', 'label': 'Identifier', @@ -51,7 +52,8 @@ def aws_assumerole_getcreds(access_key, secret_key, role_arn, external_id): if (access_key is None or len(access_key) == 0) and ( - secret_key is None or len(secret_key) == 0): + secret_key is None or len(secret_key) == 0 + ): # Connect using credentials in the EE connection = boto3.client(service_name='sts') else: @@ -59,12 +61,14 @@ def aws_assumerole_getcreds(access_key, secret_key, role_arn, external_id): connection = boto3.client( service_name='sts', aws_access_key_id=access_key, - aws_secret_access_key=secret_key) + aws_secret_access_key=secret_key, + ) try: response = connection.assume_role( RoleArn=role_arn, RoleSessionName='AAP_AWS_Role_Session1', - ExternalId=external_id) + ExternalId=external_id, + ) except ClientError as ce: raise ValueError(f'Got a bad client response from AWS: {ce.msg}.') @@ -74,7 +78,8 @@ def aws_assumerole_getcreds(access_key, secret_key, role_arn, external_id): def aws_assumerole_backend(**kwargs): - """This backend function actually contacts AWS to assume a given role for the specified user""" + """This backend function actually contacts AWS to assume a given role for + the specified user.""" access_key = kwargs.get('access_key') secret_key = kwargs.get('secret_key') role_arn = kwargs.get('role_arn') @@ -87,7 +92,8 @@ def aws_assumerole_backend(**kwargs): # multiple roles. # credential_key_hash = hashlib.sha256( - (str(access_key or '') + role_arn).encode('utf-8')) + (str(access_key or '') + role_arn).encode('utf-8'), + ) credential_key = credential_key_hash.hexdigest() credentials = _aws_cred_cache.get(credential_key, None) @@ -95,11 +101,15 @@ def aws_assumerole_backend(**kwargs): # If there are no credentials for this user/ARN *or* the credentials # we have in the cache have expired, then we need to contact AWS again. # - if (credentials is None) or (credentials['Expiration'] < datetime.datetime.now( - credentials['Expiration'].tzinfo)): + if (credentials is None) or ( + credentials['Expiration'] < datetime.datetime.now( + credentials['Expiration'].tzinfo, + ) + ): credentials = aws_assumerole_getcreds( - access_key, secret_key, role_arn, external_id) + access_key, secret_key, role_arn, external_id, + ) _aws_cred_cache[credential_key] = credentials @@ -114,4 +124,5 @@ def aws_assumerole_backend(**kwargs): aws_assumerole_plugin = CredentialPlugin( 'AWS Assume Role Plugin', inputs=assume_role_inputs, - backend=aws_assumerole_backend) + backend=aws_assumerole_backend, +) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index ec844f0be9..1169742650 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -135,71 +135,72 @@ def test_hashivault_handle_auth_not_enough_args() -> None: hashivault.handle_auth() -def test_aws_assumerole_with_accesssecret(): - ''' - Test that the aws_assumerole_backend function call returns a token given the access_key and secret_key. - ''' +def test_aws_assumerole_with_accesssecret(monkeypatch): + """Test that the aws_assumerole_backend function call returns a token given + the access_key and secret_key.""" kwargs = { 'access_key': 'my_access_key', 'secret_key': 'my_secret_key', 'role_arn': 'the_arn', 'identifier': 'access_token', } - with mock.patch.object(aws_assumerole, 'aws_assumerole_getcreds') as method_mock: - method_mock.return_value = { + + def mock_getcreds(access_key, secret_key, role_arn, session_token): + return { 'access_key': 'the_access_key', 'secret_key': 'the_secret_key', 'access_token': 'the_access_token', 'Expiration': datetime.datetime.today() + datetime.timedelta(days=1), } - token = aws_assumerole.aws_assumerole_backend(**kwargs) - method_mock.assert_called_with( - kwargs.get('access_key'), - kwargs.get('secret_key'), - kwargs.get('role_arn'), - None) - assert token == 'the_access_token' - kwargs['identifier'] = 'secret_key' - method_mock.reset_mock() - token = aws_assumerole.aws_assumerole_backend(**kwargs) - method_mock.assert_not_called() - assert token == 'the_secret_key' - kwargs['identifier'] = 'access_key' - method_mock.reset_mock() - token = aws_assumerole.aws_assumerole_backend(**kwargs) - method_mock.assert_not_called() - assert token == 'the_access_key' - - -def test_aws_assumerole_with_arnonly(): - ''' - Test backend function with only the role ARN provided. - ''' + + monkeypatch.setattr( + aws_assumerole, + 'aws_assumerole_getcreds', + mock_getcreds) + + token = aws_assumerole.aws_assumerole_backend(**kwargs) + assert token == 'the_access_token' + + kwargs['identifier'] = 'secret_key' + token = aws_assumerole.aws_assumerole_backend(**kwargs) + assert token == 'the_secret_key' + + kwargs['identifier'] = 'access_key' + token = aws_assumerole.aws_assumerole_backend(**kwargs) + assert token == 'the_access_key' + + +def test_aws_assumerole_with_arnonly(monkeypatch): + """Test backend function with only the role ARN provided.""" kwargs = { 'role_arn': 'the_arn', 'identifier': 'access_token', } - with mock.patch.object(aws_assumerole, 'aws_assumerole_getcreds') as method_mock: - method_mock.return_value = { + + # Define a mock function that will replace aws_assumerole_getcreds + def mock_getcreds(*args, **kwargs): + return { 'access_key': 'the_access_key', 'secret_key': 'the_secret_key', 'access_token': 'the_access_token', 'Expiration': datetime.datetime.today() + datetime.timedelta(days=1), } - token = aws_assumerole.aws_assumerole_backend(**kwargs) - method_mock.assert_called_with( - None, None, kwargs.get('role_arn'), None) - assert token == 'the_access_token' - kwargs['identifier'] = 'secret_key' - method_mock.reset_mock() - token = aws_assumerole.aws_assumerole_backend(**kwargs) - method_mock.assert_not_called() - assert token == 'the_secret_key' - kwargs['identifier'] = 'access_key' - method_mock.reset_mock() - token = aws_assumerole.aws_assumerole_backend(**kwargs) - method_mock.assert_not_called() - assert token == 'the_access_key' + + monkeypatch.setattr( + aws_assumerole, + 'aws_assumerole_getcreds', + mock_getcreds) + + token = aws_assumerole.aws_assumerole_backend(**kwargs) + assert token == 'the_access_token' + + kwargs['identifier'] = 'secret_key' + token = aws_assumerole.aws_assumerole_backend(**kwargs) + assert token == 'the_secret_key' + + kwargs['identifier'] = 'access_key' + token = aws_assumerole.aws_assumerole_backend(**kwargs) + assert token == 'the_access_key' class TestDelineaImports: From 35e5dcbf65f51fd8001ec4e2b69603d00b7a642a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 Sep 2024 18:07:46 +0000 Subject: [PATCH 06/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/credential_plugins_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index 1169742650..157e122c96 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -156,7 +156,8 @@ def mock_getcreds(access_key, secret_key, role_arn, session_token): monkeypatch.setattr( aws_assumerole, 'aws_assumerole_getcreds', - mock_getcreds) + mock_getcreds, + ) token = aws_assumerole.aws_assumerole_backend(**kwargs) assert token == 'the_access_token' @@ -189,7 +190,8 @@ def mock_getcreds(*args, **kwargs): monkeypatch.setattr( aws_assumerole, 'aws_assumerole_getcreds', - mock_getcreds) + mock_getcreds, + ) token = aws_assumerole.aws_assumerole_backend(**kwargs) assert token == 'the_access_token' From 656ecb78565a412393f8af045acaeb6cd3ae3ed7 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Thu, 5 Sep 2024 15:36:11 -0400 Subject: [PATCH 07/67] fix some linting and add ignores --- src/awx_plugins/credentials/aws_assumerole.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index 3aa17605af..87e73aeee6 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -11,7 +11,9 @@ try: from botocore.exceptions import ClientError except ImportError: - pass # caught by AnsibleAWSModule + """ + caught by AnsibleAWSModule + """ _aws_cred_cache = {} From 847d081d9d52b6ca787cebee0c26d07098225328 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 Sep 2024 19:37:39 +0000 Subject: [PATCH 08/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/awx_plugins/credentials/aws_assumerole.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index 87e73aeee6..f7cf0ee035 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -11,9 +11,7 @@ try: from botocore.exceptions import ClientError except ImportError: - """ - caught by AnsibleAWSModule - """ + """Caught by AnsibleAWSModule.""" _aws_cred_cache = {} From 1807b7627cd42e25e55268cef5d72bdacb383c9c Mon Sep 17 00:00:00 2001 From: Jake Jackson Date: Fri, 6 Sep 2024 16:00:06 -0400 Subject: [PATCH 09/67] update accesssecrettest to add typing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) --- tests/credential_plugins_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index 157e122c96..d90fa74868 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -135,7 +135,7 @@ def test_hashivault_handle_auth_not_enough_args() -> None: hashivault.handle_auth() -def test_aws_assumerole_with_accesssecret(monkeypatch): +def test_aws_assumerole_with_accesssecret(monkeypatch: pytest.MonkeyPatch) -> None: """Test that the aws_assumerole_backend function call returns a token given the access_key and secret_key.""" kwargs = { From 648e8f7027f2f274fb3e2cc7b6e733b2c236a1af Mon Sep 17 00:00:00 2001 From: Jake Jackson Date: Fri, 6 Sep 2024 16:00:32 -0400 Subject: [PATCH 10/67] update arnonly test with typing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) --- tests/credential_plugins_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index d90fa74868..eb1129e55f 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -171,7 +171,7 @@ def mock_getcreds(access_key, secret_key, role_arn, session_token): assert token == 'the_access_key' -def test_aws_assumerole_with_arnonly(monkeypatch): +def test_aws_assumerole_with_arnonly(monkeypatch: pytest.MonkeyPatch) -> None: """Test backend function with only the role ARN provided.""" kwargs = { 'role_arn': 'the_arn', From 4dd176a6d3d6e5fe389ebef4f2136055e58bf1b9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 20:01:04 +0000 Subject: [PATCH 11/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/credential_plugins_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index eb1129e55f..ad3bb543f4 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -135,7 +135,8 @@ def test_hashivault_handle_auth_not_enough_args() -> None: hashivault.handle_auth() -def test_aws_assumerole_with_accesssecret(monkeypatch: pytest.MonkeyPatch) -> None: +def test_aws_assumerole_with_accesssecret( + monkeypatch: pytest.MonkeyPatch) -> None: """Test that the aws_assumerole_backend function call returns a token given the access_key and secret_key.""" kwargs = { From c6a99ae723b5160d6b1fb7c4e4be43fc83496b1a Mon Sep 17 00:00:00 2001 From: Jake Jackson Date: Fri, 6 Sep 2024 16:01:53 -0400 Subject: [PATCH 12/67] remove datetime as its found in stdlib MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8fa8b8b788..73473f3dcc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,6 @@ dependencies = [ # runtime deps # https://packaging.python.org/en/latest/guide "python-dsv-sdk >= 1.0.4", # credentials.thycotic_dsv "python-tss-sdk >= 1.2.1", # credentials.thycotic_tss "requests", # credentials.aim, credentials.centrify_vault, credentials.conjur, credentials.hashivault - "datetime", # credentials.aws_assume_role ] classifiers = [ # Allowlist: https://pypi.org/classifiers/ "Development Status :: 1 - Planning", From fb2a28ce9de6c536444fde2c397e1d4ea56518f3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 20:02:50 +0000 Subject: [PATCH 13/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/credential_plugins_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index ad3bb543f4..a04ec9398f 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -136,7 +136,8 @@ def test_hashivault_handle_auth_not_enough_args() -> None: def test_aws_assumerole_with_accesssecret( - monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch: pytest.MonkeyPatch, +) -> None: """Test that the aws_assumerole_backend function call returns a token given the access_key and secret_key.""" kwargs = { From 5268832369e4cf44d01a11949d66327843968bf3 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Fri, 6 Sep 2024 16:12:36 -0400 Subject: [PATCH 14/67] update inputs field to fit formatting --- src/awx_plugins/credentials/aws_assumerole.py | 79 +++++++++++-------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index f7cf0ee035..1b1eb6e1a0 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -16,38 +16,53 @@ _aws_cred_cache = {} -assume_role_inputs = {'fields': [{'id': 'access_key', - 'label': _('AWS Access Key'), - 'type': 'string', - 'secret': True, - 'help_text': _('The optional AWS access key for the user who will assume the role'), - }, - {'id': 'secret_key', - 'label': 'AWS Secret Key', - 'type': 'string', - 'secret': True, - 'help_text': _('The optional AWS secret key for the user who will assume the role'), - }, - {'id': 'external_id', - 'label': 'External ID', - 'type': 'string', - 'help_text': _('The optional External ID which will be provided to the assume role API'), - }, - {'id': 'role_arn', - 'label': 'AWS ARN Role Name', - 'type': 'string', - 'secret': True, - 'help_text': _('The ARN Role Name to be assumed in AWS'), - }, - ], - 'metadata': [{'id': 'identifier', - 'label': 'Identifier', - 'type': 'string', - 'help_text': _('The name of the key in the assumed AWS role to fetch [AccessKeyId | SecretAccessKey | SessionToken].'), - }, - ], - 'required': ['role_arn'], - } +assume_role_inputs = { + 'fields': [ + { + 'id': 'access_key', + 'label': _('AWS Access Key'), + 'type': 'string', + 'secret': True, + 'help_text': _( + 'The optional AWS access key for' + 'the user who will assume the role.', + ), + }, + { + 'id': 'secret_key', + 'label': 'AWS Secret Key', + 'type': 'string', + 'secret': True, + 'help_text': _( + 'The optional AWS secret key for the' + 'user who will assume the role.', + ), + }, + { + 'id': 'external_id', + 'label': 'External ID', + 'type': 'string', + 'help_text': _( + 'The optional External ID which will' + 'be provided to the assume role API.', + ), + }, + ], + 'metadata': [ + { + 'id': 'identifier', + 'label': 'Identifier', + 'type': 'string', + 'help_text': _( + 'The name of the key in the assumed AWS role' + 'to fetch [AccessKeyId | SecretAccessKey | SessionToken].' + ), + }, + ], + 'required': [ + 'role_arn', + ], +} def aws_assumerole_getcreds(access_key, secret_key, role_arn, external_id): From d9435eb4395a97952170a8155ecbcba19fcf81b7 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Thu, 26 Sep 2024 11:22:54 -0400 Subject: [PATCH 15/67] update assumerole and rebase --- src/awx_plugins/credentials/aws_assumerole.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index 1b1eb6e1a0..dabbed727c 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -92,7 +92,7 @@ def aws_assumerole_getcreds(access_key, secret_key, role_arn, external_id): return credentials -def aws_assumerole_backend(**kwargs): +def aws_assumerole_backend(**kwargs) -> dict: """This backend function actually contacts AWS to assume a given role for the specified user.""" access_key = kwargs.get('access_key') From 836b5c9e7d04453f642abad6ca62144015fed7a7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:25:08 +0000 Subject: [PATCH 16/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/awx_plugins/credentials/aws_assumerole.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index dabbed727c..cc80f9d133 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -43,7 +43,7 @@ 'label': 'External ID', 'type': 'string', 'help_text': _( - 'The optional External ID which will' + 'The optional External ID which will' 'be provided to the assume role API.', ), }, @@ -55,13 +55,13 @@ 'type': 'string', 'help_text': _( 'The name of the key in the assumed AWS role' - 'to fetch [AccessKeyId | SecretAccessKey | SessionToken].' - ), + 'to fetch [AccessKeyId | SecretAccessKey | SessionToken].', + ), }, ], 'required': [ 'role_arn', - ], + ], } From e949dd20e2aa202a2411d4b1e90d4a5b3d07fd4a Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Fri, 27 Sep 2024 11:33:05 -0400 Subject: [PATCH 17/67] parameterize tests and fix more linting --- src/awx_plugins/credentials/aws_assumerole.py | 38 ++++---- tests/credential_plugins_test.py | 88 ++++++++----------- tests/importable_test.py | 6 ++ 3 files changed, 64 insertions(+), 68 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index cc80f9d133..349a1d6b02 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -1,11 +1,9 @@ import datetime import hashlib -from django.utils.translation import gettext_lazy as _ - import boto3 -from .plugin import CredentialPlugin +from .plugin import CredentialPlugin, translate_function as _ try: @@ -65,19 +63,19 @@ } -def aws_assumerole_getcreds(access_key, secret_key, role_arn, external_id): - if (access_key is None or len(access_key) == 0) and ( - secret_key is None or len(secret_key) == 0 - ): - # Connect using credentials in the EE - connection = boto3.client(service_name='sts') - else: - # Connect to AWS using provided credentials - connection = boto3.client( - service_name='sts', - aws_access_key_id=access_key, - aws_secret_access_key=secret_key, - ) +def aws_assumerole_getcreds( + access_key: str | None, + secret_key: str | None, + role_arn: str | None, + external_id: int, +) -> dict: + explicit_credentials_empty = not access_key and not secret_key + credential_kwargs = {} if explicit_credentials_empty else { + # EE creds are read from the env + 'aws_access_key_id': access_key, + 'aws_secret_access_key': secret_key, + } + connection = boto3.client(service_name='sts', **credential_kwargs) try: response = connection.assume_role( RoleArn=role_arn, @@ -130,10 +128,12 @@ def aws_assumerole_backend(**kwargs) -> dict: credentials = _aws_cred_cache.get(credential_key, None) - if identifier in credentials: + try: return credentials[identifier] - - raise ValueError(f'Could not find a value for {identifier}.') + except KeyError as key_err: + raise ValueError( + f'Could not find a value for {identifier}.', + ) from key_err aws_assumerole_plugin = CredentialPlugin( diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index a04ec9398f..064ba88313 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -135,17 +135,46 @@ def test_hashivault_handle_auth_not_enough_args() -> None: hashivault.handle_auth() -def test_aws_assumerole_with_accesssecret( +@pytest.mark.parametrize( + 'kwargs', [ + { + 'access_key': 'my_access_key', + 'secret_key': 'my_secret_key', + 'role_arn': 'the_arn', + 'identifier': 'access_token', + }, + { + 'role_arn': 'the_arn', + 'identifier': 'access_token', + }, + ], +) +@pytest.mark.parametrize( + ( + 'identifier_key', + 'expected', + ), + [ + ( + None, + 'the_access_token', + ), + ( + 'access_key', + 'the_access_key', + ), + ( + 'secret_key', + 'the_secret_key', + ), + ], +) +def test_aws_assumerole_identifier( monkeypatch: pytest.MonkeyPatch, + kwargs, identifier_key, expected, ) -> None: """Test that the aws_assumerole_backend function call returns a token given the access_key and secret_key.""" - kwargs = { - 'access_key': 'my_access_key', - 'secret_key': 'my_secret_key', - 'role_arn': 'the_arn', - 'identifier': 'access_token', - } def mock_getcreds(access_key, secret_key, role_arn, session_token): return { @@ -161,50 +190,11 @@ def mock_getcreds(access_key, secret_key, role_arn, session_token): mock_getcreds, ) - token = aws_assumerole.aws_assumerole_backend(**kwargs) - assert token == 'the_access_token' - - kwargs['identifier'] = 'secret_key' - token = aws_assumerole.aws_assumerole_backend(**kwargs) - assert token == 'the_secret_key' - - kwargs['identifier'] = 'access_key' - token = aws_assumerole.aws_assumerole_backend(**kwargs) - assert token == 'the_access_key' - - -def test_aws_assumerole_with_arnonly(monkeypatch: pytest.MonkeyPatch) -> None: - """Test backend function with only the role ARN provided.""" - kwargs = { - 'role_arn': 'the_arn', - 'identifier': 'access_token', - } - - # Define a mock function that will replace aws_assumerole_getcreds - def mock_getcreds(*args, **kwargs): - return { - 'access_key': 'the_access_key', - 'secret_key': 'the_secret_key', - 'access_token': 'the_access_token', - 'Expiration': datetime.datetime.today() + datetime.timedelta(days=1), - } - - monkeypatch.setattr( - aws_assumerole, - 'aws_assumerole_getcreds', - mock_getcreds, - ) - - token = aws_assumerole.aws_assumerole_backend(**kwargs) - assert token == 'the_access_token' - - kwargs['identifier'] = 'secret_key' - token = aws_assumerole.aws_assumerole_backend(**kwargs) - assert token == 'the_secret_key' + if identifier_key: + kwargs['identifier'] = identifier_key - kwargs['identifier'] = 'access_key' token = aws_assumerole.aws_assumerole_backend(**kwargs) - assert token == 'the_access_key' + assert token == expected class TestDelineaImports: diff --git a/tests/importable_test.py b/tests/importable_test.py index 851aef15b6..9242c705c4 100644 --- a/tests/importable_test.py +++ b/tests/importable_test.py @@ -70,6 +70,12 @@ def __str__(self) -> str: 'aws_secretsmanager_credential', 'awx_plugins.credentials.aws_secretsmanager:aws_secretmanager_plugin', ), + EntryPointParam( + 'awx_plugins.credentials', + 'aws_assumerole', + 'awx_plugins.credentials:aws_assumerole_plugin', + ), + ) From 2d48e827623d1f616ed31f09f67d63a4952a4530 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Tue, 1 Oct 2024 09:54:12 -0400 Subject: [PATCH 18/67] linter updates, update pre-commit with new deps, update expired logic --- .pre-commit-config.yaml | 9 +++-- pyproject.toml | 1 + src/awx_plugins/credentials/aws_assumerole.py | 33 +++++++++++-------- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cd26c0bcad..f16e80e268 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -197,7 +197,8 @@ repos: @ git+https://github.com/ansible/awx_plugins.interfaces.git@ad4a965 - azure-identity # needed by credentials.azure_kv - azure-keyvault # needed by credentials.azure_kv - - boto3-stubs # needed by credentials.awx_secretsmanager + - boto3-stubs[sts] # needed by credentials.awx_secretsmanager + - django-stubs # needed by credentials.injectors and inventory.plugins - lxml # dep of `--txt-report`, `--cobertura-xml-report` & `--html-report` - msrestazure # needed by credentials.azure_kv - pytest @@ -227,7 +228,8 @@ repos: @ git+https://github.com/ansible/awx_plugins.interfaces.git@ad4a965 - azure-identity # needed by credentials.azure_kv - azure-keyvault # needed by credentials.azure_kv - - boto3-stubs # needed by credentials.awx_secretsmanager + - boto3-stubs[sts] # needed by credentials.awx_secretsmanager + - django-stubs # needed by credentials.injectors and inventory.plugins - lxml # dep of `--txt-report`, `--cobertura-xml-report` & `--html-report` - msrestazure # needed by credentials.azure_kv - pytest @@ -257,7 +259,8 @@ repos: @ git+https://github.com/ansible/awx_plugins.interfaces.git@ad4a965 - azure-identity # needed by credentials.azure_kv - azure-keyvault # needed by credentials.azure_kv - - boto3-stubs # needed by credentials.awx_secretsmanager + - boto3-stubs[sts] # needed by credentials.awx_secretsmanager + - django-stubs # needed by credentials.injectors and inventory.plugins - lxml # dep of `--txt-report`, `--cobertura-xml-report` & `--html-report` - msrestazure # needed by credentials.azure_kv - pytest diff --git a/pyproject.toml b/pyproject.toml index 73473f3dcc..69bd051e01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ # runtime deps # https://packaging.python.org/en/latest/guide "python-dsv-sdk >= 1.0.4", # credentials.thycotic_dsv "python-tss-sdk >= 1.2.1", # credentials.thycotic_tss "requests", # credentials.aim, credentials.centrify_vault, credentials.conjur, credentials.hashivault + "botocore.execeptions", # credentials.aws_assume_role ] classifiers = [ # Allowlist: https://pypi.org/classifiers/ "Development Status :: 1 - Planning", diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index 349a1d6b02..5e54952d3b 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -1,16 +1,14 @@ +"""This module provides integration with AWS AssumeRole functionality.""" + import datetime import hashlib import boto3 +from botocore.exceptions import ClientError from .plugin import CredentialPlugin, translate_function as _ -try: - from botocore.exceptions import ClientError -except ImportError: - """Caught by AnsibleAWSModule.""" - _aws_cred_cache = {} @@ -69,6 +67,7 @@ def aws_assumerole_getcreds( role_arn: str | None, external_id: int, ) -> dict: + """This function gets the credentials and returns them for use.""" explicit_credentials_empty = not access_key and not secret_key credential_kwargs = {} if explicit_credentials_empty else { # EE creds are read from the env @@ -90,9 +89,15 @@ def aws_assumerole_getcreds( return credentials -def aws_assumerole_backend(**kwargs) -> dict: - """This backend function actually contacts AWS to assume a given role for - the specified user.""" +def aws_assumerole_backend( + access_key: str | None, + secret_key: str | None, + role_arn: str | None, + external_id: int, + identifier: str, +) -> dict: + """This function contacts AWS to assume a given role for the user.""" + access_key = kwargs.get('access_key') secret_key = kwargs.get('secret_key') role_arn = kwargs.get('role_arn') @@ -103,7 +108,7 @@ def aws_assumerole_backend(**kwargs) -> dict: # This should allow two users requesting the same ARN role to have # separate credentials, and should allow the same user to request # multiple roles. - # + credential_key_hash = hashlib.sha256( (str(access_key or '') + role_arn).encode('utf-8'), ) @@ -114,11 +119,11 @@ def aws_assumerole_backend(**kwargs) -> dict: # If there are no credentials for this user/ARN *or* the credentials # we have in the cache have expired, then we need to contact AWS again. # - if (credentials is None) or ( - credentials['Expiration'] < datetime.datetime.now( - credentials['Expiration'].tzinfo, - ) - ): + creds_expired = ( + (creds_expire_at := credentials.get('Expiration')) and + creds_expire_at < datetime.now(credentials['Expiration'].tzinfo) + ) + if creds_expired: credentials = aws_assumerole_getcreds( access_key, secret_key, role_arn, external_id, From ef334bc29e71ffc8a9e4692b46cbd0540a8f5ea3 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Tue, 1 Oct 2024 15:58:46 -0400 Subject: [PATCH 19/67] more linter fixes, add some ignores and fix toml dep --- .flake8 | 17 ++--- pyproject.toml | 2 +- src/awx_plugins/credentials/aws_assumerole.py | 47 ++++++++----- tests/credential_plugins_test.py | 67 ++++++++++--------- 4 files changed, 79 insertions(+), 54 deletions(-) diff --git a/.flake8 b/.flake8 index d4497132a4..69b8128614 100644 --- a/.flake8 +++ b/.flake8 @@ -111,14 +111,15 @@ per-file-ignores = tests/**.py: DAR, DCO020, S101, S105, S108, S404, S603, WPS202, WPS210, WPS430, WPS436, WPS441, WPS442, WPS450 # The following ignores must be fixed and the entries removed from this config: - src/awx_plugins/credentials/aim.py: ANN003, ANN201, B950, CCR001, D100, D103, LN001, Q003, WPS210, WPS221, WPS223, WPS231, WPS336, WPS432 - src/awx_plugins/credentials/aws_secretsmanager.py: ANN003, ANN201, D100, D103, WPS111, WPS210, WPS329, WPS529 - src/awx_plugins/credentials/azure_kv.py: ANN003, ANN201, D100, D103, WPS111, WPS361, WPS421 - src/awx_plugins/credentials/centrify_vault.py: ANN003, ANN201, D100, D103, N802, P101, WPS210, WPS229 - src/awx_plugins/credentials/conjur.py: ANN003, ANN201, B950, D100, D103, E800, P101, WPS111, WPS210, WPS229, WPS432, WPS440 - src/awx_plugins/credentials/dsv.py: ANN003, ANN201, D100, D103, P103, WPS210 - src/awx_plugins/credentials/hashivault.py: ANN003, ANN201, B950, C901, CCR001, D100, D103, LN001, N400, WPS202, WPS204, WPS210, WPS221, WPS223, WPS229, WPS231, WPS232, WPS331, WPS336, WPS337, WPS432, WPS454 - src/awx_plugins/credentials/injectors.py: ANN001, ANN201, ANN202, C408, D100, D103, WPS110, WPS111, WPS202, WPS210, WPS347, WPS433, WPS440 + src/awx_plugins/credentials/aim.py: ANN003, ANN201, B950, CCR001, D100, D103, LN001, Q003, WPS210, WPS221, WPS223, WPS226, WPS231, WPS336, WPS432 + src/awx_plugins/credentials/aws_secretsmanager.py: ANN003, ANN201, D100, D103, WPS111, WPS210, WPS226, WPS329, WPS529 + src/awx_plugins/credentials/aws_assumerole.py: WPS226, WPS332 + src/awx_plugins/credentials/azure_kv.py: ANN003, ANN201, D100, D103, WPS111, WPS226, WPS361, WPS421 + src/awx_plugins/credentials/centrify_vault.py: ANN003, ANN201, D100, D103, N802, P101, WPS210, WPS226, WPS229 + src/awx_plugins/credentials/conjur.py: ANN003, ANN201, B950, D100, D103, E800, P101, WPS111, WPS210, WPS226, WPS229, WPS432, WPS440 + src/awx_plugins/credentials/dsv.py: ANN003, ANN201, D100, D103, P103, WPS210, WPS226 + src/awx_plugins/credentials/hashivault.py: ANN003, ANN201, B950, C901, CCR001, D100, D103, LN001, N400, WPS202, WPS204, WPS210, WPS221, WPS223, WPS226, WPS229, WPS231, WPS232, WPS331, WPS336, WPS337, WPS432, WPS454 + src/awx_plugins/credentials/injectors.py: ANN001, ANN201, ANN202, C408, D100, D103, WPS110, WPS111, WPS202, WPS210, WPS226, WPS347, WPS433, WPS440 src/awx_plugins/credentials/plugin.py: ANN001, ANN002, ANN101, ANN201, ANN204, B010, D100, D101, D103, D105, D107, D205, D400, E731, WPS115, WPS432, WPS433, WPS440, WPS442, WPS601 src/awx_plugins/credentials/plugins.py: B950,D100, D101, D103, D105, D107, D205, D400, LN001, WPS204, WPS229, WPS433, WPS440 src/awx_plugins/credentials/tss.py: ANN003, ANN201, D100, D103, E712, WPS433, WPS440, WPS503 diff --git a/pyproject.toml b/pyproject.toml index 69bd051e01..2f26a29cb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ # runtime deps # https://packaging.python.org/en/latest/guide "python-dsv-sdk >= 1.0.4", # credentials.thycotic_dsv "python-tss-sdk >= 1.2.1", # credentials.thycotic_tss "requests", # credentials.aim, credentials.centrify_vault, credentials.conjur, credentials.hashivault - "botocore.execeptions", # credentials.aws_assume_role + "botocore", # credentials.aws_assume_role ] classifiers = [ # Allowlist: https://pypi.org/classifiers/ "Development Status :: 1 - Planning", diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index 5e54952d3b..9c1b81bdc6 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -67,7 +67,20 @@ def aws_assumerole_getcreds( role_arn: str | None, external_id: int, ) -> dict: - """This function gets the credentials and returns them for use.""" + """Return the credentials for use. + + :param access_key: The AWS access key ID. + :type access_key: str + :param secret_key: The AWS secret access key. + :type secret_key: str + :param role_arn: The ARN received from AWS. + :type role_arn: str + :param external_id: The external ID received from AWS. + :type external_id: int + :returns: The credentials received from AWS. + :rtype: dict + :raises ValueError: If the client response is bad. + """ explicit_credentials_empty = not access_key and not secret_key credential_kwargs = {} if explicit_credentials_empty else { # EE creds are read from the env @@ -82,11 +95,9 @@ def aws_assumerole_getcreds( ExternalId=external_id, ) except ClientError as ce: - raise ValueError(f'Got a bad client response from AWS: {ce.msg}.') + raise ValueError(f'Got a bad client response from AWS: {ce.message}.') - credentials = response.get('Credentials', {}) - - return credentials + return response.get('Credentials', {}) def aws_assumerole_backend( @@ -96,19 +107,26 @@ def aws_assumerole_backend( external_id: int, identifier: str, ) -> dict: - """This function contacts AWS to assume a given role for the user.""" - - access_key = kwargs.get('access_key') - secret_key = kwargs.get('secret_key') - role_arn = kwargs.get('role_arn') - external_id = kwargs.get('external_id') - identifier = kwargs.get('identifier') - + """Contact AWS to assume a given role for the user. + + :param access_key: The AWS access key ID. + :type access_key: str + :param secret_key: The AWS secret access key. + :type secret_key: str + :param role_arn: The ARN received from AWS. + :type role_arn: str + :param external_id: The external ID received from AWS. + :type external_id: int + :param identifier: The identifier to fetch from the assumed role. + :type identifier: str + :raises ValueError: If the identifier is not found. + :returns: The identifier fetched from the assumed role. + :rtype: dict + """ # Generate a unique SHA256 hash for combo of user access key and ARN # This should allow two users requesting the same ARN role to have # separate credentials, and should allow the same user to request # multiple roles. - credential_key_hash = hashlib.sha256( (str(access_key or '') + role_arn).encode('utf-8'), ) @@ -118,7 +136,6 @@ def aws_assumerole_backend( # If there are no credentials for this user/ARN *or* the credentials # we have in the cache have expired, then we need to contact AWS again. - # creds_expired = ( (creds_expire_at := credentials.get('Expiration')) and creds_expire_at < datetime.now(credentials['Expiration'].tzinfo) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index 064ba88313..9f33786e36 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -136,47 +136,54 @@ def test_hashivault_handle_auth_not_enough_args() -> None: @pytest.mark.parametrize( - 'kwargs', [ - { - 'access_key': 'my_access_key', - 'secret_key': 'my_secret_key', - 'role_arn': 'the_arn', - 'identifier': 'access_token', - }, - { - 'role_arn': 'the_arn', - 'identifier': 'access_token', - }, - ], + 'kwargs', ( + ( + { + 'access_key': 'my_access_key', + 'secret_key': 'my_secret_key', + 'role_arn': 'the_arn', + 'identifier': 'access_token', + }, + ), + ( + { + 'role_arn': 'the_arn', + 'identifier': 'access_token', + }, + ), + ), ) @pytest.mark.parametrize( ( 'identifier_key', 'expected', ), - [ - ( - None, - 'the_access_token', - ), - ( - 'access_key', - 'the_access_key', - ), - ( - 'secret_key', - 'the_secret_key', - ), - ], -) + ( + None, + 'the_access_token', + ), + ( + 'access_key', + 'the_access_key', + ), + ( + 'secret_key', + 'the_secret_key', + ), + ) def test_aws_assumerole_identifier( - monkeypatch: pytest.MonkeyPatch, - kwargs, identifier_key, expected, + monkeypatch: pytest.MonkeyPatch, + kwargs: dict, identifier_key: str, expected: dict[str, str], ) -> None: """Test that the aws_assumerole_backend function call returns a token given the access_key and secret_key.""" - def mock_getcreds(access_key, secret_key, role_arn, session_token): + def mock_getcreds( + access_key: str | None, + secret_key: str | None, + role_arn: str | None, + external_id: int, + ) -> dict: return { 'access_key': 'the_access_key', 'secret_key': 'the_secret_key', From 814e586998abbe4510355726348614e5b1e9602e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 20:02:21 +0000 Subject: [PATCH 20/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/credential_plugins_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index 9f33786e36..8350ae3075 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -170,7 +170,7 @@ def test_hashivault_handle_auth_not_enough_args() -> None: 'secret_key', 'the_secret_key', ), - ) +) def test_aws_assumerole_identifier( monkeypatch: pytest.MonkeyPatch, kwargs: dict, identifier_key: str, expected: dict[str, str], From 93a28bddec9fc0713daa18ead307bf75cbcaeae6 Mon Sep 17 00:00:00 2001 From: Jake Jackson Date: Wed, 2 Oct 2024 09:59:57 -0400 Subject: [PATCH 21/67] Update .flake8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) --- .flake8 | 1 - 1 file changed, 1 deletion(-) diff --git a/.flake8 b/.flake8 index 69b8128614..ceebeaec02 100644 --- a/.flake8 +++ b/.flake8 @@ -113,7 +113,6 @@ per-file-ignores = # The following ignores must be fixed and the entries removed from this config: src/awx_plugins/credentials/aim.py: ANN003, ANN201, B950, CCR001, D100, D103, LN001, Q003, WPS210, WPS221, WPS223, WPS226, WPS231, WPS336, WPS432 src/awx_plugins/credentials/aws_secretsmanager.py: ANN003, ANN201, D100, D103, WPS111, WPS210, WPS226, WPS329, WPS529 - src/awx_plugins/credentials/aws_assumerole.py: WPS226, WPS332 src/awx_plugins/credentials/azure_kv.py: ANN003, ANN201, D100, D103, WPS111, WPS226, WPS361, WPS421 src/awx_plugins/credentials/centrify_vault.py: ANN003, ANN201, D100, D103, N802, P101, WPS210, WPS226, WPS229 src/awx_plugins/credentials/conjur.py: ANN003, ANN201, B950, D100, D103, E800, P101, WPS111, WPS210, WPS226, WPS229, WPS432, WPS440 From 1345871b0dcdbcc3b9ae234af3d3dbe0b607a54f Mon Sep 17 00:00:00 2001 From: Jake Jackson Date: Wed, 2 Oct 2024 10:00:19 -0400 Subject: [PATCH 22/67] Update tests/importable_test.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) --- tests/importable_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/importable_test.py b/tests/importable_test.py index 9242c705c4..64e826d5a5 100644 --- a/tests/importable_test.py +++ b/tests/importable_test.py @@ -75,7 +75,6 @@ def __str__(self) -> str: 'aws_assumerole', 'awx_plugins.credentials:aws_assumerole_plugin', ), - ) From a0737ef143110a26d9867a11ece2406dcd6f8207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Thu, 3 Oct 2024 20:53:20 +0200 Subject: [PATCH 23/67] Fix pytest params nesting --- tests/credential_plugins_test.py | 42 +++++++++++++------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index 8350ae3075..df5d4297a9 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -136,21 +136,20 @@ def test_hashivault_handle_auth_not_enough_args() -> None: @pytest.mark.parametrize( - 'kwargs', ( - ( - { - 'access_key': 'my_access_key', - 'secret_key': 'my_secret_key', - 'role_arn': 'the_arn', - 'identifier': 'access_token', - }, - ), - ( - { - 'role_arn': 'the_arn', - 'identifier': 'access_token', - }, - ), + 'kwargs', + ( + { + 'access_key': 'my_access_key', + 'secret_key': 'my_secret_key', + 'role_arn': 'the_arn', + 'identifier': 'access_token', + }, + ), + ( + { + 'role_arn': 'the_arn', + 'identifier': 'access_token', + }, ), ) @pytest.mark.parametrize( @@ -159,16 +158,9 @@ def test_hashivault_handle_auth_not_enough_args() -> None: 'expected', ), ( - None, - 'the_access_token', - ), - ( - 'access_key', - 'the_access_key', - ), - ( - 'secret_key', - 'the_secret_key', + (None, 'the_access_token'), + ('access_key', 'the_access_key'), + ('secret_key', 'the_secret_key'), ), ) def test_aws_assumerole_identifier( From 29aee1e62a44b0f6800b2ec2421497a1091ec166 Mon Sep 17 00:00:00 2001 From: Jake Jackson Date: Mon, 7 Oct 2024 14:38:34 -0400 Subject: [PATCH 24/67] Update pyproject.toml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2f26a29cb4..e63db24377 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ # runtime deps # https://packaging.python.org/en/latest/guide "python-dsv-sdk >= 1.0.4", # credentials.thycotic_dsv "python-tss-sdk >= 1.2.1", # credentials.thycotic_tss "requests", # credentials.aim, credentials.centrify_vault, credentials.conjur, credentials.hashivault - "botocore", # credentials.aws_assume_role + "botocore", # credentials.aws_assume_role, credentials.awx_secretsmanager ] classifiers = [ # Allowlist: https://pypi.org/classifiers/ "Development Status :: 1 - Planning", From 86eef44f1e0a2b20ed958b965848371f9b7984cc Mon Sep 17 00:00:00 2001 From: Jake Jackson Date: Mon, 7 Oct 2024 14:41:59 -0400 Subject: [PATCH 25/67] Update src/awx_plugins/credentials/aws_assumerole.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) --- src/awx_plugins/credentials/aws_assumerole.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index 9c1b81bdc6..48b055daa9 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -94,8 +94,10 @@ def aws_assumerole_getcreds( RoleSessionName='AAP_AWS_Role_Session1', ExternalId=external_id, ) - except ClientError as ce: - raise ValueError(f'Got a bad client response from AWS: {ce.message}.') + except ClientError as client_err: + raise ValueError( + f'Got a bad client response from AWS: {ce.message}.', + ) from client_err return response.get('Credentials', {}) From ea9d0e786b4128de65602a9d0178b1c266b1b426 Mon Sep 17 00:00:00 2001 From: Jake Jackson Date: Mon, 7 Oct 2024 20:20:44 -0400 Subject: [PATCH 26/67] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) --- src/awx_plugins/credentials/aws_assumerole.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index 48b055daa9..c970c3f193 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -81,15 +81,18 @@ def aws_assumerole_getcreds( :rtype: dict :raises ValueError: If the client response is bad. """ - explicit_credentials_empty = not access_key and not secret_key - credential_kwargs = {} if explicit_credentials_empty else { - # EE creds are read from the env - 'aws_access_key_id': access_key, - 'aws_secret_access_key': secret_key, - } - connection = boto3.client(service_name='sts', **credential_kwargs) + from mypy_boto3_sts.client import STSClient + + connection: STSClient = boto3.client( + service_name='sts', + # The following EE creds are read from the env if they are not passed: + aws_access_key_id=access_key, # defaults to `None` in the lib + aws_secret_access_key=secret_key, # defaults to `None` in the lib + ) try: - response = connection.assume_role( + from mypy_boto3_sts.type_defs import AssumeRoleResponseTypeDef + + response: AssumeRoleResponseTypeDef = connection.assume_role( RoleArn=role_arn, RoleSessionName='AAP_AWS_Role_Session1', ExternalId=external_id, From f1e40353c6ad0b25c039054a2e554df66fc4e1b9 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Tue, 1 Oct 2024 15:58:46 -0400 Subject: [PATCH 27/67] more linter fixes, add some ignores and fix toml dep --- .flake8 | 1 + tests/credential_plugins_test.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index ceebeaec02..69b8128614 100644 --- a/.flake8 +++ b/.flake8 @@ -113,6 +113,7 @@ per-file-ignores = # The following ignores must be fixed and the entries removed from this config: src/awx_plugins/credentials/aim.py: ANN003, ANN201, B950, CCR001, D100, D103, LN001, Q003, WPS210, WPS221, WPS223, WPS226, WPS231, WPS336, WPS432 src/awx_plugins/credentials/aws_secretsmanager.py: ANN003, ANN201, D100, D103, WPS111, WPS210, WPS226, WPS329, WPS529 + src/awx_plugins/credentials/aws_assumerole.py: WPS226, WPS332 src/awx_plugins/credentials/azure_kv.py: ANN003, ANN201, D100, D103, WPS111, WPS226, WPS361, WPS421 src/awx_plugins/credentials/centrify_vault.py: ANN003, ANN201, D100, D103, N802, P101, WPS210, WPS226, WPS229 src/awx_plugins/credentials/conjur.py: ANN003, ANN201, B950, D100, D103, E800, P101, WPS111, WPS210, WPS226, WPS229, WPS432, WPS440 diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index df5d4297a9..5bc9986c86 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -162,7 +162,7 @@ def test_hashivault_handle_auth_not_enough_args() -> None: ('access_key', 'the_access_key'), ('secret_key', 'the_secret_key'), ), -) + ) def test_aws_assumerole_identifier( monkeypatch: pytest.MonkeyPatch, kwargs: dict, identifier_key: str, expected: dict[str, str], From 4e7cc436e049ae10dacde8a7748a0eacf82e0539 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Mon, 7 Oct 2024 20:55:07 -0400 Subject: [PATCH 28/67] rebase and include fixes from sviat --- .flake8 | 1 + tests/credential_plugins_test.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.flake8 b/.flake8 index 69b8128614..fc2d7f0672 100644 --- a/.flake8 +++ b/.flake8 @@ -126,6 +126,7 @@ per-file-ignores = src/awx_plugins/inventory/plugins.py: ANN001, ANN002, ANN003, ANN101, ANN102, ANN201, ANN202, ANN206, B950, C812, C819, D100, D101, D102, D205, D209, D400, D401, LN001, LN002, N801, WPS110, WPS111, WPS202, WPS210, WPS214, WPS301, WPS319, WPS324, WPS331, WPS336, WPS337, WPS338, WPS347, WPS421, WPS433, WPS450, WPS510, WPS529 tests/credential_plugins_test.py: ANN101, B017, C419, D100, D102, D103, D205, D209, D400, DAR, PT011, S105, WPS111, WPS117, WPS118, WPS202, WPS352, WPS421, WPS433, WPS507 tests/importable_test.py: ANN101, DAR + tests/credential_plugins_test.py: DAR, WPS430 # Count the number of occurrences of each error/warning code and print a report: statistics = true diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index 5bc9986c86..8478311545 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -158,11 +158,11 @@ def test_hashivault_handle_auth_not_enough_args() -> None: 'expected', ), ( - (None, 'the_access_token'), - ('access_key', 'the_access_key'), - ('secret_key', 'the_secret_key'), + (None, 'the_access_token'), + ('access_key', 'the_access_key'), + ('secret_key', 'the_secret_key'), ), - ) +) def test_aws_assumerole_identifier( monkeypatch: pytest.MonkeyPatch, kwargs: dict, identifier_key: str, expected: dict[str, str], From 499c4943ec62408dac01e05f9a129a31a794578e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Tue, 8 Oct 2024 14:47:27 +0200 Subject: [PATCH 29/67] Move cred plugin test rule ignores to the first flake8 entry --- .flake8 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.flake8 b/.flake8 index fc2d7f0672..070b331964 100644 --- a/.flake8 +++ b/.flake8 @@ -124,9 +124,8 @@ per-file-ignores = src/awx_plugins/credentials/plugins.py: B950,D100, D101, D103, D105, D107, D205, D400, LN001, WPS204, WPS229, WPS433, WPS440 src/awx_plugins/credentials/tss.py: ANN003, ANN201, D100, D103, E712, WPS433, WPS440, WPS503 src/awx_plugins/inventory/plugins.py: ANN001, ANN002, ANN003, ANN101, ANN102, ANN201, ANN202, ANN206, B950, C812, C819, D100, D101, D102, D205, D209, D400, D401, LN001, LN002, N801, WPS110, WPS111, WPS202, WPS210, WPS214, WPS301, WPS319, WPS324, WPS331, WPS336, WPS337, WPS338, WPS347, WPS421, WPS433, WPS450, WPS510, WPS529 - tests/credential_plugins_test.py: ANN101, B017, C419, D100, D102, D103, D205, D209, D400, DAR, PT011, S105, WPS111, WPS117, WPS118, WPS202, WPS352, WPS421, WPS433, WPS507 + tests/credential_plugins_test.py: ANN101, B017, C419, D100, D102, D103, D205, D209, D400, DAR, PT011, S105, WPS111, WPS117, WPS118, WPS202, WPS352, WPS421, WPS430, WPS433, WPS507 tests/importable_test.py: ANN101, DAR - tests/credential_plugins_test.py: DAR, WPS430 # Count the number of occurrences of each error/warning code and print a report: statistics = true From 87cd1820589996071e1e88e6d644d7b309ec00a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Tue, 8 Oct 2024 14:48:10 +0200 Subject: [PATCH 30/67] Import gettext_noop from the interfaces dep --- src/awx_plugins/credentials/aws_assumerole.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index c970c3f193..6e4e6f88d6 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -6,7 +6,11 @@ import boto3 from botocore.exceptions import ClientError -from .plugin import CredentialPlugin, translate_function as _ +from awx_plugins.interfaces._temporary_private_django_api import ( # noqa: WPS436 + gettext_noop as _, +) + +from .plugin import CredentialPlugin _aws_cred_cache = {} From 5d346ff71a9339abc3aecbcaedcb90b347cfceed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Tue, 8 Oct 2024 14:48:38 +0200 Subject: [PATCH 31/67] Reference existing `client_err` in exception handling --- src/awx_plugins/credentials/aws_assumerole.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index 6e4e6f88d6..e697c898a8 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -103,7 +103,7 @@ def aws_assumerole_getcreds( ) except ClientError as client_err: raise ValueError( - f'Got a bad client response from AWS: {ce.message}.', + f'Got a bad client response from AWS: {client_err.message}.', ) from client_err return response.get('Credentials', {}) From 3f4340c8615d3f4f359d0654fd2545ce37aca88a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:50:24 +0000 Subject: [PATCH 32/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/awx_plugins/credentials/aws_assumerole.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index e697c898a8..bb3988e144 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -3,13 +3,13 @@ import datetime import hashlib -import boto3 -from botocore.exceptions import ClientError - from awx_plugins.interfaces._temporary_private_django_api import ( # noqa: WPS436 gettext_noop as _, ) +import boto3 +from botocore.exceptions import ClientError + from .plugin import CredentialPlugin From 3e44826a301a686f09168c844b3b6b108534a615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Tue, 8 Oct 2024 14:53:23 +0200 Subject: [PATCH 33/67] Mention that boto3 is also a dep of the new plugin --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e63db24377..b8e9633390 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ # runtime deps # https://packaging.python.org/en/latest/guide "PyYAML", # credentials.injectors, inventory.plugins "azure-identity", # credentials.azure_kv "azure-keyvault", # credentials.azure_kv - "boto3", # credentials.awx_secretsmanager + "boto3", # credentials.aws_assume_role, credentials.awx_secretsmanager "msrestazure", # credentials.azure_kv "python-dsv-sdk >= 1.0.4", # credentials.thycotic_dsv "python-tss-sdk >= 1.2.1", # credentials.thycotic_tss From b308b67de11278802ed84bea109dc67270d41ed6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Tue, 8 Oct 2024 14:53:55 +0200 Subject: [PATCH 34/67] List botocore after boto3 in the deps --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b8e9633390..854c514bd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,11 +23,11 @@ dependencies = [ # runtime deps # https://packaging.python.org/en/latest/guide "azure-identity", # credentials.azure_kv "azure-keyvault", # credentials.azure_kv "boto3", # credentials.aws_assume_role, credentials.awx_secretsmanager + "botocore", # credentials.aws_assume_role, credentials.awx_secretsmanager "msrestazure", # credentials.azure_kv "python-dsv-sdk >= 1.0.4", # credentials.thycotic_dsv "python-tss-sdk >= 1.2.1", # credentials.thycotic_tss "requests", # credentials.aim, credentials.centrify_vault, credentials.conjur, credentials.hashivault - "botocore", # credentials.aws_assume_role, credentials.awx_secretsmanager ] classifiers = [ # Allowlist: https://pypi.org/classifiers/ "Development Status :: 1 - Planning", From a132eeb358c8d3ba501a4de362a047f17ce67657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Tue, 8 Oct 2024 15:38:30 +0200 Subject: [PATCH 35/67] Fix kwargs variants definition in tests --- tests/credential_plugins_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index 8478311545..08357af28e 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -144,8 +144,6 @@ def test_hashivault_handle_auth_not_enough_args() -> None: 'role_arn': 'the_arn', 'identifier': 'access_token', }, - ), - ( { 'role_arn': 'the_arn', 'identifier': 'access_token', From d4415710bc135b9c7b9df9fcded68c2cea4cd1b0 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Tue, 8 Oct 2024 10:25:22 -0400 Subject: [PATCH 36/67] fix imports and prep for consuming sviat suggestions --- src/awx_plugins/credentials/aws_assumerole.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index bb3988e144..ba7b4c7d31 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -9,6 +9,12 @@ import boto3 from botocore.exceptions import ClientError +import typing + + +if typing.TYPE_CHECKING: + from mypy_boto3_sts.client import STSClient + from mypy_boto3_sts.type_defs import AssumeRoleResponseTypeDef from .plugin import CredentialPlugin @@ -85,8 +91,6 @@ def aws_assumerole_getcreds( :rtype: dict :raises ValueError: If the client response is bad. """ - from mypy_boto3_sts.client import STSClient - connection: STSClient = boto3.client( service_name='sts', # The following EE creds are read from the env if they are not passed: @@ -94,8 +98,6 @@ def aws_assumerole_getcreds( aws_secret_access_key=secret_key, # defaults to `None` in the lib ) try: - from mypy_boto3_sts.type_defs import AssumeRoleResponseTypeDef - response: AssumeRoleResponseTypeDef = connection.assume_role( RoleArn=role_arn, RoleSessionName='AAP_AWS_Role_Session1', From 9883c3d922ffcbfcd54ebf2199efa350a726d089 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:26:55 +0000 Subject: [PATCH 37/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/awx_plugins/credentials/aws_assumerole.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index ba7b4c7d31..fc86333462 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -2,6 +2,7 @@ import datetime import hashlib +import typing from awx_plugins.interfaces._temporary_private_django_api import ( # noqa: WPS436 gettext_noop as _, @@ -9,7 +10,6 @@ import boto3 from botocore.exceptions import ClientError -import typing if typing.TYPE_CHECKING: From b565e0ffbb51ab64f5086919401b7d6739969b82 Mon Sep 17 00:00:00 2001 From: Jake Jackson Date: Tue, 8 Oct 2024 10:27:41 -0400 Subject: [PATCH 38/67] Apply suggestions from code review from @webknjaz MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) --- src/awx_plugins/credentials/aws_assumerole.py | 6 +++--- tests/credential_plugins_test.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index fc86333462..7e78edc81d 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -19,7 +19,7 @@ from .plugin import CredentialPlugin -_aws_cred_cache = {} +_aws_cred_cache: dict[str, CredentialsTypeDef | dict[Never, Never]] | dict[Never, Never] = {} assume_role_inputs = { @@ -76,7 +76,7 @@ def aws_assumerole_getcreds( secret_key: str | None, role_arn: str | None, external_id: int, -) -> dict: +) -> CredentialsTypeDef | dict[Never, Never]: """Return the credentials for use. :param access_key: The AWS access key ID. @@ -159,7 +159,7 @@ def aws_assumerole_backend( _aws_cred_cache[credential_key] = credentials - credentials = _aws_cred_cache.get(credential_key, None) + credentials = _aws_cred_cache.get(credential_key, {}) try: return credentials[identifier] diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index 08357af28e..1051779731 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -163,7 +163,7 @@ def test_hashivault_handle_auth_not_enough_args() -> None: ) def test_aws_assumerole_identifier( monkeypatch: pytest.MonkeyPatch, - kwargs: dict, identifier_key: str, expected: dict[str, str], + kwargs: dict[str, str], identifier_key: str | None, expected: str, ) -> None: """Test that the aws_assumerole_backend function call returns a token given the access_key and secret_key.""" @@ -187,10 +187,9 @@ def mock_getcreds( mock_getcreds, ) - if identifier_key: - kwargs['identifier'] = identifier_key + extra_kwargs = {'identifier': identifier_key} if identifier_key else {} - token = aws_assumerole.aws_assumerole_backend(**kwargs) + token = aws_assumerole.aws_assumerole_backend(**kwargs, **extra_kwargs) assert token == expected From e876ff06c30a36283594f837e67d56227acc9ab6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:29:06 +0000 Subject: [PATCH 39/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/awx_plugins/credentials/aws_assumerole.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index 7e78edc81d..d1702e1d74 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -19,7 +19,8 @@ from .plugin import CredentialPlugin -_aws_cred_cache: dict[str, CredentialsTypeDef | dict[Never, Never]] | dict[Never, Never] = {} +_aws_cred_cache: dict[str, CredentialsTypeDef | + dict[Never, Never]] | dict[Never, Never] = {} assume_role_inputs = { From bf1ec349c3e404697ea5b8de4a4b8f927cbb9ed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Wed, 9 Oct 2024 02:54:54 +0200 Subject: [PATCH 40/67] Use `Never` from `typing` --- src/awx_plugins/credentials/aws_assumerole.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index d1702e1d74..9d196186b6 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -19,8 +19,10 @@ from .plugin import CredentialPlugin -_aws_cred_cache: dict[str, CredentialsTypeDef | - dict[Never, Never]] | dict[Never, Never] = {} +_aws_cred_cache: dict[ + str, + CredentialsTypeDef | dict[typing.Never, typing.Never], +] | dict[typing.Never, typing.Never] = {} assume_role_inputs = { @@ -77,7 +79,7 @@ def aws_assumerole_getcreds( secret_key: str | None, role_arn: str | None, external_id: int, -) -> CredentialsTypeDef | dict[Never, Never]: +) -> CredentialsTypeDef | dict[typing.Never, typing.Never]: """Return the credentials for use. :param access_key: The AWS access key ID. From 753b6af009423785ffeb9df36ecc7289a93f1519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Wed, 9 Oct 2024 03:02:53 +0200 Subject: [PATCH 41/67] Import missing `CredentialsTypeDef` --- src/awx_plugins/credentials/aws_assumerole.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index 9d196186b6..98a647630d 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -14,7 +14,7 @@ if typing.TYPE_CHECKING: from mypy_boto3_sts.client import STSClient - from mypy_boto3_sts.type_defs import AssumeRoleResponseTypeDef + from mypy_boto3_sts.type_defs import AssumeRoleResponseTypeDef, CredentialsTypeDef from .plugin import CredentialPlugin From d5f99d69f414dab739ccbb10faa68b1777862e10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Wed, 9 Oct 2024 03:05:10 +0200 Subject: [PATCH 42/67] Make the import line fit the limit --- src/awx_plugins/credentials/aws_assumerole.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index 98a647630d..cb81170999 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -14,7 +14,10 @@ if typing.TYPE_CHECKING: from mypy_boto3_sts.client import STSClient - from mypy_boto3_sts.type_defs import AssumeRoleResponseTypeDef, CredentialsTypeDef + from mypy_boto3_sts.type_defs import ( + AssumeRoleResponseTypeDef, + CredentialsTypeDef, + ) from .plugin import CredentialPlugin From b1a2129d693aa8c52aba1835af2b45dee82cdda6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Wed, 9 Oct 2024 03:10:17 +0200 Subject: [PATCH 43/67] Quote conditionally imported types An alternative would be to add a `from __future__ import annotations` for lazy evaluation. --- src/awx_plugins/credentials/aws_assumerole.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index cb81170999..ccf1e034a7 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -24,7 +24,7 @@ _aws_cred_cache: dict[ str, - CredentialsTypeDef | dict[typing.Never, typing.Never], + "CredentialsTypeDef" | dict[typing.Never, typing.Never], ] | dict[typing.Never, typing.Never] = {} @@ -82,7 +82,7 @@ def aws_assumerole_getcreds( secret_key: str | None, role_arn: str | None, external_id: int, -) -> CredentialsTypeDef | dict[typing.Never, typing.Never]: +) -> "CredentialsTypeDef" | dict[typing.Never, typing.Never]: """Return the credentials for use. :param access_key: The AWS access key ID. @@ -97,14 +97,14 @@ def aws_assumerole_getcreds( :rtype: dict :raises ValueError: If the client response is bad. """ - connection: STSClient = boto3.client( + connection: "STSClient" = boto3.client( service_name='sts', # The following EE creds are read from the env if they are not passed: aws_access_key_id=access_key, # defaults to `None` in the lib aws_secret_access_key=secret_key, # defaults to `None` in the lib ) try: - response: AssumeRoleResponseTypeDef = connection.assume_role( + response: "AssumeRoleResponseTypeDef" = connection.assume_role( RoleArn=role_arn, RoleSessionName='AAP_AWS_Role_Session1', ExternalId=external_id, From 94303339a5d3dcfeef4fd581da39003b2e4eeca8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 9 Oct 2024 01:11:16 +0000 Subject: [PATCH 44/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/awx_plugins/credentials/aws_assumerole.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index ccf1e034a7..7c3207c548 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -24,7 +24,7 @@ _aws_cred_cache: dict[ str, - "CredentialsTypeDef" | dict[typing.Never, typing.Never], + 'CredentialsTypeDef' | dict[typing.Never, typing.Never], ] | dict[typing.Never, typing.Never] = {} @@ -82,7 +82,7 @@ def aws_assumerole_getcreds( secret_key: str | None, role_arn: str | None, external_id: int, -) -> "CredentialsTypeDef" | dict[typing.Never, typing.Never]: +) -> 'CredentialsTypeDef' | dict[typing.Never, typing.Never]: """Return the credentials for use. :param access_key: The AWS access key ID. @@ -97,14 +97,14 @@ def aws_assumerole_getcreds( :rtype: dict :raises ValueError: If the client response is bad. """ - connection: "STSClient" = boto3.client( + connection: 'STSClient' = boto3.client( service_name='sts', # The following EE creds are read from the env if they are not passed: aws_access_key_id=access_key, # defaults to `None` in the lib aws_secret_access_key=secret_key, # defaults to `None` in the lib ) try: - response: "AssumeRoleResponseTypeDef" = connection.assume_role( + response: 'AssumeRoleResponseTypeDef' = connection.assume_role( RoleArn=role_arn, RoleSessionName='AAP_AWS_Role_Session1', ExternalId=external_id, From e93e91722b095842b9d8a8db0fae4c11f4103431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Wed, 9 Oct 2024 03:14:40 +0200 Subject: [PATCH 45/67] Quote type concats --- src/awx_plugins/credentials/aws_assumerole.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index 7c3207c548..d51bbf63b4 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -24,7 +24,7 @@ _aws_cred_cache: dict[ str, - 'CredentialsTypeDef' | dict[typing.Never, typing.Never], + 'CredentialsTypeDef | dict[typing.Never, typing.Never]', ] | dict[typing.Never, typing.Never] = {} @@ -82,7 +82,7 @@ def aws_assumerole_getcreds( secret_key: str | None, role_arn: str | None, external_id: int, -) -> 'CredentialsTypeDef' | dict[typing.Never, typing.Never]: +) -> 'CredentialsTypeDef | dict[typing.Never, typing.Never]': """Return the credentials for use. :param access_key: The AWS access key ID. From 4ada473e595abe2161fa5bd3d061f2264fe1f418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Wed, 9 Oct 2024 03:40:12 +0200 Subject: [PATCH 46/67] Fix boto3 type refs in docs --- docs/conf.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index d219194bf4..1480073169 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -7,6 +7,13 @@ from pathlib import Path from tomllib import loads as _parse_toml +from sphinx.addnodes import pending_xref +from sphinx.application import Sphinx +from sphinx.environment import BuildEnvironment + +# isort: split + +from docutils.nodes import literal, reference # -- Path setup -------------------------------------------------------------- @@ -195,3 +202,42 @@ nitpick_ignore = [ # temporarily listed ('role', 'reference') pairs that Sphinx cannot resolve ] + + +def _replace_missing_boto3_reference( + app: Sphinx, + env: BuildEnvironment, + node: pending_xref, + contnode: literal, +) -> reference | None: + if (node.get('refdomain'), node.get('reftype')) != ('py', 'class'): + return None + + boto3_type_uri_map = { + 'AssumeRoleResponseTypeDef': 'type_defs/#assumeroleresponsetypedef', + 'CredentialsTypeDef': 'type_defs/#credentialstypedef', + 'STSClient': 'client/#stsclient', + } + ref_target = node.get('reftarget', '') + + try: + return reference( + ref_target, + ref_target, + internal=False, + refuri=f'https://youtype.github.io/boto3_stubs_docs/mypy_boto3_sts/{boto3_type_uri_map[ref_target]}', + ) + except KeyError: + return None + + +def setup(app: Sphinx) -> dict[str, bool | str]: + """Register project-local Sphinx extension-API customizations.""" + + app.connect('missing-reference', _replace_missing_boto3_reference) + + return { + 'parallel_read_safe': True, + 'parallel_write_safe': True, + 'version': release, + } From 0b1e5ef85aeffd7e694647a022a04416701d5d63 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 9 Oct 2024 01:41:26 +0000 Subject: [PATCH 47/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/conf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 1480073169..da8f07593c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,10 +11,12 @@ from sphinx.application import Sphinx from sphinx.environment import BuildEnvironment + # isort: split from docutils.nodes import literal, reference + # -- Path setup -------------------------------------------------------------- DOCS_ROOT_DIR = Path(__file__).parent.resolve() @@ -225,7 +227,8 @@ def _replace_missing_boto3_reference( ref_target, ref_target, internal=False, - refuri=f'https://youtype.github.io/boto3_stubs_docs/mypy_boto3_sts/{boto3_type_uri_map[ref_target]}', + refuri=f'https://youtype.github.io/boto3_stubs_docs/mypy_boto3_sts/{ + boto3_type_uri_map[ref_target]}', ) except KeyError: return None From fb85b7e857925e57a92ff4248d541d99e053989f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Wed, 9 Oct 2024 03:43:16 +0200 Subject: [PATCH 48/67] Unbreak the docstring --- docs/conf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index da8f07593c..313b27e95e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -227,8 +227,9 @@ def _replace_missing_boto3_reference( ref_target, ref_target, internal=False, - refuri=f'https://youtype.github.io/boto3_stubs_docs/mypy_boto3_sts/{ - boto3_type_uri_map[ref_target]}', + refuri= + 'https://youtype.github.io/boto3_stubs_docs/' + f'mypy_boto3_sts/{boto3_type_uri_map[ref_target]}', ) except KeyError: return None From 444d12173bad5768aebee82e7f7d9db9334b1067 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 9 Oct 2024 01:44:14 +0000 Subject: [PATCH 49/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/conf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 313b27e95e..9a64ed61ce 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -227,8 +227,7 @@ def _replace_missing_boto3_reference( ref_target, ref_target, internal=False, - refuri= - 'https://youtype.github.io/boto3_stubs_docs/' + refuri='https://youtype.github.io/boto3_stubs_docs/' f'mypy_boto3_sts/{boto3_type_uri_map[ref_target]}', ) except KeyError: From 74b92062b6d15f0da4ded93058a60e79bf455980 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Wed, 9 Oct 2024 03:50:34 +0200 Subject: [PATCH 50/67] Fix DAR @ Sphinx config --- docs/conf.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9a64ed61ce..4558f209b0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -235,8 +235,14 @@ def _replace_missing_boto3_reference( def setup(app: Sphinx) -> dict[str, bool | str]: - """Register project-local Sphinx extension-API customizations.""" - + """Register project-local Sphinx extension-API customizations. + + :param app: Initialized Sphinx app instance. + :type app: Sphinx + + :returns: Extension metadata. + :rtype: dict[str, bool | str] + """ app.connect('missing-reference', _replace_missing_boto3_reference) return { From c4dd94dbd66d58e27900fe0943ac2bb5767d0aa1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 9 Oct 2024 01:51:32 +0000 Subject: [PATCH 51/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/conf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 4558f209b0..1cfd3a744d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -236,10 +236,9 @@ def _replace_missing_boto3_reference( def setup(app: Sphinx) -> dict[str, bool | str]: """Register project-local Sphinx extension-API customizations. - + :param app: Initialized Sphinx app instance. :type app: Sphinx - :returns: Extension metadata. :rtype: dict[str, bool | str] """ From 50fe83efcefac7f48bdeabcffdbeb425fd6b54b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Wed, 9 Oct 2024 03:57:15 +0200 Subject: [PATCH 52/67] Install `botocore` type stubs in addition to `boto3` --- .pre-commit-config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f16e80e268..c0125654ac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -198,6 +198,7 @@ repos: - azure-identity # needed by credentials.azure_kv - azure-keyvault # needed by credentials.azure_kv - boto3-stubs[sts] # needed by credentials.awx_secretsmanager + - botocore-stubs # needed by credentials.awx_secretsmanager - django-stubs # needed by credentials.injectors and inventory.plugins - lxml # dep of `--txt-report`, `--cobertura-xml-report` & `--html-report` - msrestazure # needed by credentials.azure_kv @@ -229,6 +230,7 @@ repos: - azure-identity # needed by credentials.azure_kv - azure-keyvault # needed by credentials.azure_kv - boto3-stubs[sts] # needed by credentials.awx_secretsmanager + - botocore-stubs # needed by credentials.awx_secretsmanager - django-stubs # needed by credentials.injectors and inventory.plugins - lxml # dep of `--txt-report`, `--cobertura-xml-report` & `--html-report` - msrestazure # needed by credentials.azure_kv @@ -260,6 +262,7 @@ repos: - azure-identity # needed by credentials.azure_kv - azure-keyvault # needed by credentials.azure_kv - boto3-stubs[sts] # needed by credentials.awx_secretsmanager + - botocore-stubs # needed by credentials.awx_secretsmanager - django-stubs # needed by credentials.injectors and inventory.plugins - lxml # dep of `--txt-report`, `--cobertura-xml-report` & `--html-report` - msrestazure # needed by credentials.azure_kv From 803c6592a24f32add75792a71956e762ea958f6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Wed, 9 Oct 2024 04:15:07 +0200 Subject: [PATCH 53/67] Merge the kwargs dicts --- tests/credential_plugins_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index 1051779731..2927ee8699 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -189,7 +189,7 @@ def mock_getcreds( extra_kwargs = {'identifier': identifier_key} if identifier_key else {} - token = aws_assumerole.aws_assumerole_backend(**kwargs, **extra_kwargs) + token = aws_assumerole.aws_assumerole_backend(**{**kwargs, **extra_kwargs}) assert token == expected From 83ab18e7824f718e85b1b1533b18e4199abd1ec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Wed, 9 Oct 2024 20:20:12 +0200 Subject: [PATCH 54/67] Simplify extra kwargs --- tests/credential_plugins_test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index 2927ee8699..efe9c2670f 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -187,9 +187,7 @@ def mock_getcreds( mock_getcreds, ) - extra_kwargs = {'identifier': identifier_key} if identifier_key else {} - - token = aws_assumerole.aws_assumerole_backend(**{**kwargs, **extra_kwargs}) + token = aws_assumerole.aws_assumerole_backend(**kwargs, identifier=identifier_key or kwargs['identifier']) assert token == expected From 3f8bb9e28ba807ac2b0ee0183ea8c979415b1735 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 9 Oct 2024 18:21:33 +0000 Subject: [PATCH 55/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/credential_plugins_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index efe9c2670f..eda3d71c35 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -187,7 +187,8 @@ def mock_getcreds( mock_getcreds, ) - token = aws_assumerole.aws_assumerole_backend(**kwargs, identifier=identifier_key or kwargs['identifier']) + token = aws_assumerole.aws_assumerole_backend( + **kwargs, identifier=identifier_key or kwargs['identifier']) assert token == expected From 669ba01ea6afe65962765ca9e9dca2f57c26b6d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Wed, 9 Oct 2024 20:23:09 +0200 Subject: [PATCH 56/67] Drop static kwargs from params --- tests/credential_plugins_test.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index eda3d71c35..141242c247 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -141,13 +141,8 @@ def test_hashivault_handle_auth_not_enough_args() -> None: { 'access_key': 'my_access_key', 'secret_key': 'my_secret_key', - 'role_arn': 'the_arn', - 'identifier': 'access_token', - }, - { - 'role_arn': 'the_arn', - 'identifier': 'access_token', }, + {}, ), ) @pytest.mark.parametrize( @@ -188,7 +183,10 @@ def mock_getcreds( ) token = aws_assumerole.aws_assumerole_backend( - **kwargs, identifier=identifier_key or kwargs['identifier']) + identifier=identifier_key or 'access_token', + role_arn='the_arn', + **kwargs, + ) assert token == expected From 881976f5ec82d7065326f02d86da2a4da0143380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Wed, 9 Oct 2024 20:24:48 +0200 Subject: [PATCH 57/67] Clarify that kwargs are actually creds --- tests/credential_plugins_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index 141242c247..c726be2d9a 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -136,7 +136,7 @@ def test_hashivault_handle_auth_not_enough_args() -> None: @pytest.mark.parametrize( - 'kwargs', + 'explicit_creds', ( { 'access_key': 'my_access_key', @@ -158,7 +158,7 @@ def test_hashivault_handle_auth_not_enough_args() -> None: ) def test_aws_assumerole_identifier( monkeypatch: pytest.MonkeyPatch, - kwargs: dict[str, str], identifier_key: str | None, expected: str, + explicit_creds: dict[str, str], identifier_key: str | None, expected: str, ) -> None: """Test that the aws_assumerole_backend function call returns a token given the access_key and secret_key.""" @@ -185,7 +185,7 @@ def mock_getcreds( token = aws_assumerole.aws_assumerole_backend( identifier=identifier_key or 'access_token', role_arn='the_arn', - **kwargs, + **explicit_creds, ) assert token == expected From 53fb4a21d4366414fc3aff88169b37da118e6d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Wed, 9 Oct 2024 20:28:17 +0200 Subject: [PATCH 58/67] Add ids to assumerole id tests --- tests/credential_plugins_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index c726be2d9a..bc2b3e61b4 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -143,6 +143,7 @@ def test_hashivault_handle_auth_not_enough_args() -> None: 'secret_key': 'my_secret_key', }, {}, + ids=('with-creds-args', 'with-env-creds'), ), ) @pytest.mark.parametrize( @@ -155,6 +156,11 @@ def test_hashivault_handle_auth_not_enough_args() -> None: ('access_key', 'the_access_key'), ('secret_key', 'the_secret_key'), ), + ids=( + 'access-token', + 'access-key', + 'secret-key', + ), ) def test_aws_assumerole_identifier( monkeypatch: pytest.MonkeyPatch, From f7db82c56e22f220b6b78dd25fe758b52767efb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Wed, 9 Oct 2024 20:29:14 +0200 Subject: [PATCH 59/67] Fixup: pass ids to the parametrize decorator --- tests/credential_plugins_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index bc2b3e61b4..5fedfaa386 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -143,8 +143,8 @@ def test_hashivault_handle_auth_not_enough_args() -> None: 'secret_key': 'my_secret_key', }, {}, - ids=('with-creds-args', 'with-env-creds'), ), + ids=('with-creds-args', 'with-env-creds'), ) @pytest.mark.parametrize( ( From 9842a9b0ba97989da13ebb694a4d78c989cac215 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Wed, 9 Oct 2024 14:55:32 -0400 Subject: [PATCH 60/67] update types, fix imports to resolve modules not found --- src/awx_plugins/credentials/aws_assumerole.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index d51bbf63b4..95551ac8fe 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -1,8 +1,9 @@ """This module provides integration with AWS AssumeRole functionality.""" -import datetime +from .plugin import CredentialPlugin import hashlib import typing +from datetime import datetime from awx_plugins.interfaces._temporary_private_django_api import ( # noqa: WPS436 gettext_noop as _, @@ -78,9 +79,9 @@ def aws_assumerole_getcreds( - access_key: str | None, - secret_key: str | None, - role_arn: str | None, + access_key: str, + secret_key: str, + role_arn: str, external_id: int, ) -> 'CredentialsTypeDef | dict[typing.Never, typing.Never]': """Return the credentials for use. @@ -118,9 +119,9 @@ def aws_assumerole_getcreds( def aws_assumerole_backend( - access_key: str | None, - secret_key: str | None, - role_arn: str | None, + access_key: str, + secret_key: str, + role_arn: str, external_id: int, identifier: str, ) -> dict: From c75d33ce9f4f6efd57be41215364098f1d71c461 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 9 Oct 2024 19:03:57 +0000 Subject: [PATCH 61/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/awx_plugins/credentials/aws_assumerole.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index 95551ac8fe..37c5d8c939 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -1,6 +1,5 @@ """This module provides integration with AWS AssumeRole functionality.""" -from .plugin import CredentialPlugin import hashlib import typing from datetime import datetime @@ -12,6 +11,8 @@ import boto3 from botocore.exceptions import ClientError +from .plugin import CredentialPlugin + if typing.TYPE_CHECKING: from mypy_boto3_sts.client import STSClient From ff886aa85f10a38ac3f4291ecf86a4aa9d7a21a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Thu, 10 Oct 2024 02:25:02 +0200 Subject: [PATCH 62/67] Drop the unnecessary django stubs --- .pre-commit-config.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c0125654ac..3e9cbb9f1a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -199,7 +199,6 @@ repos: - azure-keyvault # needed by credentials.azure_kv - boto3-stubs[sts] # needed by credentials.awx_secretsmanager - botocore-stubs # needed by credentials.awx_secretsmanager - - django-stubs # needed by credentials.injectors and inventory.plugins - lxml # dep of `--txt-report`, `--cobertura-xml-report` & `--html-report` - msrestazure # needed by credentials.azure_kv - pytest @@ -231,7 +230,6 @@ repos: - azure-keyvault # needed by credentials.azure_kv - boto3-stubs[sts] # needed by credentials.awx_secretsmanager - botocore-stubs # needed by credentials.awx_secretsmanager - - django-stubs # needed by credentials.injectors and inventory.plugins - lxml # dep of `--txt-report`, `--cobertura-xml-report` & `--html-report` - msrestazure # needed by credentials.azure_kv - pytest @@ -263,7 +261,6 @@ repos: - azure-keyvault # needed by credentials.azure_kv - boto3-stubs[sts] # needed by credentials.awx_secretsmanager - botocore-stubs # needed by credentials.awx_secretsmanager - - django-stubs # needed by credentials.injectors and inventory.plugins - lxml # dep of `--txt-report`, `--cobertura-xml-report` & `--html-report` - msrestazure # needed by credentials.azure_kv - pytest From 0dcc43050ce0a6ee75420163a85b2c42835236be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Thu, 10 Oct 2024 02:25:39 +0200 Subject: [PATCH 63/67] Drop the duplicate cred plugin import --- src/awx_plugins/credentials/aws_assumerole.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index 37c5d8c939..395c94f650 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -11,8 +11,6 @@ import boto3 from botocore.exceptions import ClientError -from .plugin import CredentialPlugin - if typing.TYPE_CHECKING: from mypy_boto3_sts.client import STSClient From 0eb6e86433132a51e7d22032ec3cadfeb771ce9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Thu, 10 Oct 2024 02:26:21 +0200 Subject: [PATCH 64/67] Keep none creds args expected --- src/awx_plugins/credentials/aws_assumerole.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index 395c94f650..ba69d24b0a 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -78,8 +78,8 @@ def aws_assumerole_getcreds( - access_key: str, - secret_key: str, + access_key: str | None, + secret_key: str | None, role_arn: str, external_id: int, ) -> 'CredentialsTypeDef | dict[typing.Never, typing.Never]': @@ -118,8 +118,8 @@ def aws_assumerole_getcreds( def aws_assumerole_backend( - access_key: str, - secret_key: str, + access_key: str | None, + secret_key: str | None, role_arn: str, external_id: int, identifier: str, From 164b54a300cbed70f2f126ca6109dd0fdd4b8d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sviatoslav=20Sydorenko=20=28=D0=A1=D0=B2=D1=8F=D1=82=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B0=D0=B2=20=D0=A1=D0=B8=D0=B4=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE=29?= Date: Thu, 10 Oct 2024 02:28:46 +0200 Subject: [PATCH 65/67] Pass the required external id in tests --- tests/credential_plugins_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index 5fedfaa386..e7dc0017dd 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -189,6 +189,7 @@ def mock_getcreds( ) token = aws_assumerole.aws_assumerole_backend( + external_id=42, identifier=identifier_key or 'access_token', role_arn='the_arn', **explicit_creds, From db9e869a82f49d7776d9baae7469944d4fd7f084 Mon Sep 17 00:00:00 2001 From: Jake Jackson Date: Thu, 10 Oct 2024 10:11:18 -0400 Subject: [PATCH 66/67] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) --- src/awx_plugins/credentials/aws_assumerole.py | 2 +- tests/credential_plugins_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/awx_plugins/credentials/aws_assumerole.py b/src/awx_plugins/credentials/aws_assumerole.py index ba69d24b0a..58d216369b 100644 --- a/src/awx_plugins/credentials/aws_assumerole.py +++ b/src/awx_plugins/credentials/aws_assumerole.py @@ -149,7 +149,7 @@ def aws_assumerole_backend( ) credential_key = credential_key_hash.hexdigest() - credentials = _aws_cred_cache.get(credential_key, None) + credentials = _aws_cred_cache.get(credential_key, {}) # If there are no credentials for this user/ARN *or* the credentials # we have in the cache have expired, then we need to contact AWS again. diff --git a/tests/credential_plugins_test.py b/tests/credential_plugins_test.py index e7dc0017dd..34c61becf6 100644 --- a/tests/credential_plugins_test.py +++ b/tests/credential_plugins_test.py @@ -172,7 +172,7 @@ def test_aws_assumerole_identifier( def mock_getcreds( access_key: str | None, secret_key: str | None, - role_arn: str | None, + role_arn: str, external_id: int, ) -> dict: return { From cbeea1ca44db8e4fa88a4be7c8be4dae2daa4142 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Tue, 1 Oct 2024 09:54:12 -0400 Subject: [PATCH 67/67] linter updates, update pre-commit with new deps, update expired logic --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 854c514bd7..616a34efe7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ # runtime deps # https://packaging.python.org/en/latest/guide "python-dsv-sdk >= 1.0.4", # credentials.thycotic_dsv "python-tss-sdk >= 1.2.1", # credentials.thycotic_tss "requests", # credentials.aim, credentials.centrify_vault, credentials.conjur, credentials.hashivault + "botocore.execeptions", # credentials.aws_assume_role ] classifiers = [ # Allowlist: https://pypi.org/classifiers/ "Development Status :: 1 - Planning",