From b186f119488f2ccc7750c5599771d96022b6e60c Mon Sep 17 00:00:00 2001 From: maxnus Date: Sun, 27 May 2018 18:46:04 +0700 Subject: [PATCH 1/5] - Used boto3 instead of botocore --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ca19a45..367a26e 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ include_package_data=True, packages=find_packages(), install_requires=[ - 'botocore>=1.2.0', + 'boto3', ], extras_require={ }, From a446c6493f586356644292000892f512e2425271 Mon Sep 17 00:00:00 2001 From: maxnus Date: Sun, 27 May 2018 18:46:25 +0700 Subject: [PATCH 2/5] - Added support for private s3 --- remotecv_aws/bucket.py | 23 ++++++++++++++++------- remotecv_aws/loader.py | 5 ++++- 2 files changed, 20 insertions(+), 8 deletions(-) mode change 100644 => 100755 remotecv_aws/loader.py diff --git a/remotecv_aws/bucket.py b/remotecv_aws/bucket.py index f669957..d4d1e58 100644 --- a/remotecv_aws/bucket.py +++ b/remotecv_aws/bucket.py @@ -4,34 +4,43 @@ # Use of this source code is governed by the MIT license that can be # found in the LICENSE file. -import botocore.session +import boto3 class Bucket(object): """ This handles all communication with AWS API """ _bucket = None - _region = None _local_cache = dict() - def __init__(self, bucket, region): + def __init__(self, bucket, region, accessKeyId, secretAccessKey, endPoint): """ Constructor :param string bucket: The bucket name :param string region: The AWS API region to use + :param string accessKeyId: The AWS access key ID for accessing the bucket + :param string secretAccessKey: The AWS secret access key for accessing the bucket + :param string endPoint: The AWS bucket custom end point :return: The created bucket """ self._bucket = bucket - self._region = region - session = None or botocore.session.get_session() - self._client = session.create_client('s3', region_name=self._region, endpoint_url=None) + + session = boto3.session.Session() + + self._client = session.client( + service_name='s3', + aws_access_key_id=accessKeyId, + aws_secret_access_key=secretAccessKey, + region_name=region, + endpoint_url=endPoint, + ) def get(self, path): """ Returns object at given path :param string path: Path or 'key' to retrieve AWS object """ - file_path = self._client.get_object( Bucket=self._bucket, Key=path) + file_path = self._client.Object( Bucket=self._bucket, Key=path) return file_path['Body'].read() diff --git a/remotecv_aws/loader.py b/remotecv_aws/loader.py old mode 100644 new mode 100755 index 99ed0e1..c8d418d --- a/remotecv_aws/loader.py +++ b/remotecv_aws/loader.py @@ -14,6 +14,9 @@ def load_sync(path): """ bucket = os.environ.get('AWS_LOADER_BUCKET') region = os.environ.get('AWS_REGION', 'eu-west-1') - bucket_loader = Bucket(bucket, region) + accessKeyId = os.environ.get('AWS_ACCESS_KEY_ID') + secretAccessKey = os.environ.get('AWS_SECRET_KEY_ID') + endPoint = os.environ.get('AWS_END_POINT') + bucket_loader = Bucket(bucket, region, accessKeyId, secretAccessKey, endPoint) return bucket_loader.get(path) \ No newline at end of file From f5dde1410e5f8593a59a99f62f5d8b5c6e05351f Mon Sep 17 00:00:00 2001 From: maxnus Date: Sun, 27 May 2018 19:19:50 +0700 Subject: [PATCH 3/5] - Fix bugs --- remotecv_aws/bucket.py | 22 +++++++++++----------- remotecv_aws/loader.py | 3 +-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/remotecv_aws/bucket.py b/remotecv_aws/bucket.py index d4d1e58..46fc7aa 100644 --- a/remotecv_aws/bucket.py +++ b/remotecv_aws/bucket.py @@ -4,7 +4,7 @@ # Use of this source code is governed by the MIT license that can be # found in the LICENSE file. -import boto3 +from boto3.session import Session class Bucket(object): """ @@ -13,7 +13,7 @@ class Bucket(object): _bucket = None _local_cache = dict() - def __init__(self, bucket, region, accessKeyId, secretAccessKey, endPoint): + def __init__(self, bucket, region, accessKeyId, secretAccessKey): """ Constructor :param string bucket: The bucket name @@ -25,22 +25,22 @@ def __init__(self, bucket, region, accessKeyId, secretAccessKey, endPoint): """ self._bucket = bucket - session = boto3.session.Session() + session = Session( + aws_access_key_id=accessKeyId, + aws_secret_access_key=secretAccessKey, + region_name=region + ) - self._client = session.client( - service_name='s3', - aws_access_key_id=accessKeyId, - aws_secret_access_key=secretAccessKey, - region_name=region, - endpoint_url=endPoint, - ) + self._client = session.resource('s3') def get(self, path): """ Returns object at given path :param string path: Path or 'key' to retrieve AWS object """ - file_path = self._client.Object( Bucket=self._bucket, Key=path) + + file_path = self._client.Bucket(self._bucket).Object(path).get() + return file_path['Body'].read() diff --git a/remotecv_aws/loader.py b/remotecv_aws/loader.py index c8d418d..e9c2798 100755 --- a/remotecv_aws/loader.py +++ b/remotecv_aws/loader.py @@ -16,7 +16,6 @@ def load_sync(path): region = os.environ.get('AWS_REGION', 'eu-west-1') accessKeyId = os.environ.get('AWS_ACCESS_KEY_ID') secretAccessKey = os.environ.get('AWS_SECRET_KEY_ID') - endPoint = os.environ.get('AWS_END_POINT') - bucket_loader = Bucket(bucket, region, accessKeyId, secretAccessKey, endPoint) + bucket_loader = Bucket(bucket, region, accessKeyId, secretAccessKey) return bucket_loader.get(path) \ No newline at end of file From 96e5461b91bd03e62e5d1f6b69507f5b479d325d Mon Sep 17 00:00:00 2001 From: maxnus Date: Sun, 27 May 2018 19:23:32 +0700 Subject: [PATCH 4/5] - Updated to 0.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 367a26e..3f24950 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name='remotecv_aws', - version='0.1', + version='0.2', description='Remotecv AWS loader', author='Edu Heraiz @ APSL', author_email='gshark@gmail.com', From 654dd628d19278fa6be4f0831880d8122d1b61ed Mon Sep 17 00:00:00 2001 From: maxnus Date: Tue, 29 May 2018 13:04:32 +0700 Subject: [PATCH 5/5] - Updated __init__ doc --- remotecv_aws/bucket.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/remotecv_aws/bucket.py b/remotecv_aws/bucket.py index 46fc7aa..e670458 100644 --- a/remotecv_aws/bucket.py +++ b/remotecv_aws/bucket.py @@ -20,8 +20,7 @@ def __init__(self, bucket, region, accessKeyId, secretAccessKey): :param string region: The AWS API region to use :param string accessKeyId: The AWS access key ID for accessing the bucket :param string secretAccessKey: The AWS secret access key for accessing the bucket - :param string endPoint: The AWS bucket custom end point - :return: The created bucket + :return: The created AWS client for the bucket """ self._bucket = bucket