From 0815db595b1963b53711e1594e059e005cd36069 Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sat, 21 Aug 2021 19:36:40 +0300 Subject: [PATCH 01/20] Add httpx to optional requirements --- requirements-optional.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-optional.txt b/requirements-optional.txt index b7d210a..f39c637 100644 --- a/requirements-optional.txt +++ b/requirements-optional.txt @@ -1,5 +1,5 @@ -requests -Werkzeug +httpx +paramiko requests-aws4auth >= 0.9 requests-aliyun >= 0.2.5 -paramiko +Werkzeug \ No newline at end of file From 1beeee1c8409f5d25b45055f9e411d1efba88da6 Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 22 Aug 2021 17:22:51 +0300 Subject: [PATCH 02/20] Replace requests with httpx --- sqlalchemy_media/stores/s3.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sqlalchemy_media/stores/s3.py b/sqlalchemy_media/stores/s3.py index 2e95c34..445c5ca 100644 --- a/sqlalchemy_media/stores/s3.py +++ b/sqlalchemy_media/stores/s3.py @@ -2,9 +2,9 @@ # Importing optional stuff required by http based store try: - import requests + import httpx except ImportError: # pragma: no cover - requests = None + httpx = None # Importing optional stuff required by S3 store @@ -38,7 +38,7 @@ class S3Store(Store): def __init__(self, bucket: str, access_key: str, secret_key: str, region: str, max_age: int = DEFAULT_MAX_AGE, prefix: str = None, base_url: str = None, - cdn_url: str = None, cdn_prefix_ignore: bool = False, + cdn_url: str = None, cdn_prefix_ignore: bool = False, acl: str = 'private'): self.bucket = bucket self.access_key = access_key @@ -68,7 +68,7 @@ def __init__(self, bucket: str, access_key: str, secret_key: str, def _get_s3_url(self, filename: str): return '{0}/{1}'.format(self.base_url, filename) - def _upload_file(self, url: str, data: str, content_type: str, + def _upload_file(self, url: str, content: str, content_type: str, rrs: bool = False): ensure_aws4auth() @@ -84,23 +84,23 @@ def _upload_file(self, url: str, data: str, content_type: str, } if content_type: headers['Content-Type'] = content_type - res = requests.put(url, auth=auth, data=data, headers=headers) + res = httpx.put(url, auth=auth, content=content, headers=headers) if not 200 <= res.status_code < 300: raise S3Error(res.text) def put(self, filename: str, stream: FileLike): url = self._get_s3_url(filename) - data = stream.read() + content = stream.read() content_type = getattr(stream, 'content_type', None) rrs = getattr(stream, 'reproducible', False) - self._upload_file(url, data, content_type, rrs=rrs) - return len(data) + self._upload_file(url, content, content_type, rrs=rrs) + return len(content) def delete(self, filename: str): ensure_aws4auth() url = self._get_s3_url(filename) auth = AWS4Auth(self.access_key, self.secret_key, self.region, 's3') - res = requests.delete(url, auth=auth) + res = httpx.delete(url, auth=auth) if not 200 <= res.status_code < 300: raise S3Error(res.text) @@ -108,7 +108,7 @@ def open(self, filename: str, mode: str = 'rb') -> FileLike: ensure_aws4auth() url = self._get_s3_url(filename) auth = AWS4Auth(self.access_key, self.secret_key, self.region, 's3') - res = requests.get(url, auth=auth) + res = httpx.get(url, auth=auth) if not 200 <= res.status_code < 300: raise S3Error(res.text) return BytesIO(res.content) From 2336821c435f8d97b5fe3e48cb8eacf1b726bc99 Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 22 Aug 2021 17:25:16 +0300 Subject: [PATCH 03/20] Replace requests with httpx for mocked s3 server --- sqlalchemy_media/tests/helpers/s3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlalchemy_media/tests/helpers/s3.py b/sqlalchemy_media/tests/helpers/s3.py index 4f6853e..beca23d 100644 --- a/sqlalchemy_media/tests/helpers/s3.py +++ b/sqlalchemy_media/tests/helpers/s3.py @@ -1,7 +1,7 @@ import contextlib from wsgiref.simple_server import WSGIRequestHandler, WSGIServer -import requests +import httpx from .http import simple_http_server @@ -20,6 +20,6 @@ def mockup_s3_server(bucket, **kwargs): url = 'http://localhost:%s' % server.server_address[1] # Create the bucket bucket_uri = '%s/%s' % (url, bucket) - res = requests.put(bucket_uri) + res = httpx.put(bucket_uri) assert res.status_code == 200 yield server, bucket_uri From 51e63193dca47b9869f7c12f70f7048afa730866 Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 22 Aug 2021 17:41:57 +0300 Subject: [PATCH 04/20] Add a standalone S3 server --- docker-compose.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1654681 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +version: "3.8" + +services: + storage: + image: minio/minio:latest + entrypoint: sh + command: -c "mkdir -p /data/test-bucket && minio server /data --console-address \":9001\"" + volumes: + - ./data:/data + ports: + - 9000:9000 + - 9001:9001 + environment: + - MINIO_ROOT_USER=test_access_key + - MINIO_ROOT_PASSWORD=test_secret_key + - MINIO_REGION_NAME=ap-northeast-2 From 9426a260b8b09d0b3b5bd6e77f68db4baa98d8d8 Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 22 Aug 2021 17:44:19 +0300 Subject: [PATCH 05/20] Replace the value for base url --- sqlalchemy_media/tests/helpers/testcases.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlalchemy_media/tests/helpers/testcases.py b/sqlalchemy_media/tests/helpers/testcases.py index b212372..940101c 100644 --- a/sqlalchemy_media/tests/helpers/testcases.py +++ b/sqlalchemy_media/tests/helpers/testcases.py @@ -26,7 +26,7 @@ def create_all_and_get_session(self): bind=self.engine, autoflush=False, autocommit=False, - expire_on_commit=True, + expire_on_commit=False, twophase=False ) return self.session_factory() @@ -54,7 +54,7 @@ def setUp(self): self.__class__.__name__, self._testMethodName ) - self.base_url = 'http://static1.example.orm' + self.base_url = 'http://localhost:9000' # Remove previous files, if any! to make a clean temp directory: if exists(self.temp_path): # pragma: no cover From d8416d05da5a797cd397350726c1da527df4f24d Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 22 Aug 2021 17:46:44 +0300 Subject: [PATCH 06/20] Change the value for base_url property in setup step --- sqlalchemy_media/tests/helpers/testcases.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlalchemy_media/tests/helpers/testcases.py b/sqlalchemy_media/tests/helpers/testcases.py index 940101c..b212372 100644 --- a/sqlalchemy_media/tests/helpers/testcases.py +++ b/sqlalchemy_media/tests/helpers/testcases.py @@ -26,7 +26,7 @@ def create_all_and_get_session(self): bind=self.engine, autoflush=False, autocommit=False, - expire_on_commit=False, + expire_on_commit=True, twophase=False ) return self.session_factory() @@ -54,7 +54,7 @@ def setUp(self): self.__class__.__name__, self._testMethodName ) - self.base_url = 'http://localhost:9000' + self.base_url = 'http://static1.example.orm' # Remove previous files, if any! to make a clean temp directory: if exists(self.temp_path): # pragma: no cover From 4b159ff7c3037b6a7c13a16ffd226383af89cf5d Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 22 Aug 2021 17:51:42 +0300 Subject: [PATCH 07/20] Change the value for base_url property in setup step --- sqlalchemy_media/tests/helpers/testcases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlalchemy_media/tests/helpers/testcases.py b/sqlalchemy_media/tests/helpers/testcases.py index b212372..1365c57 100644 --- a/sqlalchemy_media/tests/helpers/testcases.py +++ b/sqlalchemy_media/tests/helpers/testcases.py @@ -54,7 +54,7 @@ def setUp(self): self.__class__.__name__, self._testMethodName ) - self.base_url = 'http://static1.example.orm' + self.base_url = 'http://localhost:9000' # Remove previous files, if any! to make a clean temp directory: if exists(self.temp_path): # pragma: no cover From 56c53a70a9b5782379b713b54995711cac9fbdf2 Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 22 Aug 2021 17:52:54 +0300 Subject: [PATCH 08/20] Change the value for expected links in tests for image --- sqlalchemy_media/tests/test_image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlalchemy_media/tests/test_image.py b/sqlalchemy_media/tests/test_image.py index e67cb08..8281a9e 100644 --- a/sqlalchemy_media/tests/test_image.py +++ b/sqlalchemy_media/tests/test_image.py @@ -243,11 +243,11 @@ class Person(self.Base): person1 = session.query(Person).filter(Person.id == person1.id).one() with StoreManager(session): self.assertTrue(person1.image.locate().startswith( - 'http://static1.example.orm/images/image-' + 'http://localhost:9000/images/image-' )) thumbnail = person1.image.get_thumbnail(width=100) self.assertTrue(thumbnail.locate().startswith( - 'http://static1.example.orm/thumbnails/thumbnail-' + 'http://localhost:9000/thumbnails/thumbnail-' )) def test_image_list(self): From 9ff9a5a09a3d9a1e635c65efed2e99f1dcd851d7 Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 22 Aug 2021 17:54:19 +0300 Subject: [PATCH 09/20] Reffactor tests for s3 integration --- sqlalchemy_media/tests/test_s3_store.py | 526 ++++++++++++------------ 1 file changed, 257 insertions(+), 269 deletions(-) diff --git a/sqlalchemy_media/tests/test_s3_store.py b/sqlalchemy_media/tests/test_s3_store.py index 2622f04..7a0da9c 100644 --- a/sqlalchemy_media/tests/test_s3_store.py +++ b/sqlalchemy_media/tests/test_s3_store.py @@ -38,170 +38,164 @@ def setUpClass(cls): cls.this_dir = abspath(dirname(__file__)) cls.stuff_path = join(cls.this_dir, 'stuff') cls.dog_jpeg = join(cls.stuff_path, 'dog.jpg') - cls.base_url = 'http://static1.example.orm' + cls.base_url = 'http://localhost:9000/test-bucket' cls.sample_text_file1 = join(cls.stuff_path, 'sample_text_file1.txt') def test_put_from_stream(self): - with mockup_s3_server(TEST_BUCKET) as (server, uri): - store = create_s3_store(base_url=uri) - target_filename = 'test_put_from_stream/file_from_stream1.txt' - content = b'Lorem ipsum dolor sit amet' - stream = io.BytesIO(content) - length = store.put(target_filename, stream) - self.assertEqual(length, len(content)) - self.assertIsInstance(store.open(target_filename), io.BytesIO) + store = create_s3_store(base_url=self.base_url) + target_filename = 'test_put_from_stream/file_from_stream1.txt' + content = b'Lorem ipsum dolor sit amet' + stream = io.BytesIO(content) + length = store.put(target_filename, stream) + self.assertEqual(length, len(content)) + self.assertIsInstance(store.open(target_filename), io.BytesIO) def test_put_error(self): - with mockup_s3_server(TEST_BUCKET) as (server, uri): - store = create_s3_store(base_url=uri[:-2]) - target_filename = 'test_put_from_stream/file_from_stream1.txt' - content = b'Lorem ipsum dolor sit amet' - stream = io.BytesIO(content) + store = create_s3_store(base_url=self.base_url[:-2]) + target_filename = 'test_put_from_stream/file_from_stream1.txt' + content = b'Lorem ipsum dolor sit amet' + stream = io.BytesIO(content) - with self.assertRaises(S3Error): - store.put(target_filename, stream) + with self.assertRaises(S3Error): + store.put(target_filename, stream) def test_rrs_put(self): - with mockup_s3_server(TEST_BUCKET) as (server, uri): - StoreManager.register( - 's3', - functools.partial(create_s3_store, base_url=uri), - default=True - ) + StoreManager.register( + 's3', + functools.partial(create_s3_store, base_url=self.base_url), + default=True + ) - class Thumbnail(BaseThumbnail): - __reproducible__ = True + class Thumbnail(BaseThumbnail): + __reproducible__ = True - class Image(BaseImage): - __thumbnail_type__ = Thumbnail + class Image(BaseImage): + __thumbnail_type__ = Thumbnail - class Person(self.Base): - __tablename__ = 'person' - id = Column(Integer, primary_key=True) - image = Column(Image.as_mutable(Json)) + class Person(self.Base): + __tablename__ = 'person' + id = Column(Integer, primary_key=True) + image = Column(Image.as_mutable(Json)) - session = self.create_all_and_get_session() + session = self.create_all_and_get_session() - person1 = Person() - self.assertIsNone(person1.image) + person1 = Person() + self.assertIsNone(person1.image) - with StoreManager(session): - person1 = Person() - person1.image = Image.create_from(self.dog_jpeg) - self.assertIsInstance(person1.image, Image) + with StoreManager(session): + person1 = Person() + person1.image = Image.create_from(self.dog_jpeg) + self.assertIsInstance(person1.image, Image) - thumbnail = person1.image.get_thumbnail( - width=100, - auto_generate=True - ) - self.assertIsInstance(thumbnail, Thumbnail) - self.assertTrue(thumbnail.reproducible, True) + thumbnail = person1.image.get_thumbnail( + width=100, + auto_generate=True + ) + self.assertIsInstance(thumbnail, Thumbnail) + self.assertTrue(thumbnail.reproducible, True) def test_delete(self): - with mockup_s3_server(TEST_BUCKET) as (server, uri): - store = create_s3_store(base_url=uri) - target_filename = 'test_delete/sample_text_file1.txt' - with open(self.sample_text_file1, 'rb') as f: - length = store.put(target_filename, f) - self.assertEqual(length, getsize(self.sample_text_file1)) - self.assertIsInstance(store.open(target_filename), io.BytesIO) + store = create_s3_store(base_url=self.base_url) + target_filename = 'test_delete/sample_text_file1.txt' + with open(self.sample_text_file1, 'rb') as f: + length = store.put(target_filename, f) + self.assertEqual(length, getsize(self.sample_text_file1)) + self.assertIsInstance(store.open(target_filename), io.BytesIO) - store.delete(target_filename) + store.delete(target_filename) - with self.assertRaises(S3Error): - store.open(target_filename) + with self.assertRaises(S3Error): + store.open(target_filename) def test_delete_error(self): - with mockup_s3_server(TEST_BUCKET) as (server, uri): - store = create_s3_store(base_url=uri) - wrong_store = create_s3_store(base_url=uri[:-2]) - target_filename = 'test_delete/sample_text_file1.txt' - with open(self.sample_text_file1, 'rb') as f: - length = store.put(target_filename, f) - self.assertEqual(length, getsize(self.sample_text_file1)) - self.assertIsInstance(store.open(target_filename), io.BytesIO) - - with self.assertRaises(S3Error): - wrong_store.delete(target_filename) + store = create_s3_store(base_url=self.base_url) + wrong_store = create_s3_store(base_url=self.base_url[:-2]) + target_filename = 'test_delete/sample_text_file1.txt' + with open(self.sample_text_file1, 'rb') as f: + length = store.put(target_filename, f) + self.assertEqual(length, getsize(self.sample_text_file1)) + self.assertIsInstance(store.open(target_filename), io.BytesIO) + + with self.assertRaises(S3Error): + wrong_store.delete(target_filename) def test_open(self): - with mockup_s3_server(TEST_BUCKET) as (server, uri): - store = create_s3_store(base_url=uri) - target_filename = 'test_delete/sample_text_file1.txt' - with open(self.sample_text_file1, 'rb') as f: - length = store.put(target_filename, f) - self.assertEqual(length, getsize(self.sample_text_file1)) - self.assertIsInstance(store.open(target_filename), io.BytesIO) - - # Reading - with store.open(target_filename, mode='rb') as stored_file, \ - open(self.sample_text_file1, mode='rb') as original_file: - self.assertEqual(stored_file.read(), original_file.read()) + store = create_s3_store(base_url=self.base_url) + target_filename = 'test_delete/sample_text_file1.txt' + with open(self.sample_text_file1, 'rb') as f: + length = store.put(target_filename, f) + self.assertEqual(length, getsize(self.sample_text_file1)) + self.assertIsInstance(store.open(target_filename), io.BytesIO) + + # Reading + with store.open(target_filename, mode='rb') as stored_file, \ + open(self.sample_text_file1, mode='rb') as original_file: + self.assertEqual(stored_file.read(), original_file.read()) def test_locate(self): - with mockup_s3_server(TEST_BUCKET) as (server, uri): - StoreManager.register( - 's3', - functools.partial(create_s3_store, base_url=uri), - default=True - ) + StoreManager.register( + 's3', + functools.partial(create_s3_store, base_url=self.base_url), + default=True + ) + + class Person(self.Base): + __tablename__ = 'person' + id = Column(Integer, primary_key=True) + file = Column(File.as_mutable(Json)) - class Person(self.Base): - __tablename__ = 'person' - id = Column(Integer, primary_key=True) - file = Column(File.as_mutable(Json)) + session = self.create_all_and_get_session() - session = self.create_all_and_get_session() + person1 = Person() + self.assertIsNone(person1.file) + sample_content = b'Simple text.' + with StoreManager(session): person1 = Person() - self.assertIsNone(person1.file) - sample_content = b'Simple text.' - - with StoreManager(session): - person1 = Person() - person1.file = File.create_from(io.BytesIO(sample_content), - content_type='text/plain', - extension='.txt') - self.assertIsInstance(person1.file, File) - self.assertEqual( - person1.file.locate(), - '%s/%s?_ts=%s' % ( - uri, person1.file.path, person1.file.timestamp - ) + person1.file = File.create_from(io.BytesIO(sample_content), + content_type='text/plain', + extension='.txt') + self.assertIsInstance(person1.file, File) + self.assertEqual( + person1.file.locate(), + '%s/%s?_ts=%s' % ( + self.base_url, person1.file.path, person1.file.timestamp ) + ) def test_prefix(self): - with mockup_s3_server(TEST_BUCKET) as (server, uri): - prefix = 'test' - StoreManager.register( - 's3', - functools.partial(create_s3_store, base_url=uri, prefix=prefix), - default=True - ) - class Person(self.Base): - __tablename__ = 'person' - id = Column(Integer, primary_key=True) - file = Column(File.as_mutable(Json)) + prefix = 'test' + StoreManager.register( + 's3', + functools.partial( + create_s3_store, base_url=self.base_url, prefix=prefix), + default=True + ) + + class Person(self.Base): + __tablename__ = 'person' + id = Column(Integer, primary_key=True) + file = Column(File.as_mutable(Json)) + + session = self.create_all_and_get_session() - session = self.create_all_and_get_session() + person1 = Person() + self.assertIsNone(person1.file) + sample_content = b'Simple text.' + with StoreManager(session): person1 = Person() - self.assertIsNone(person1.file) - sample_content = b'Simple text.' - - with StoreManager(session): - person1 = Person() - person1.file = File.create_from(io.BytesIO(sample_content), - content_type='text/plain', - extension='.txt') - self.assertIsInstance(person1.file, File) - self.assertEqual( - person1.file.locate(), - '%s/%s/%s?_ts=%s' % ( - uri, prefix, person1.file.path, person1.file.timestamp - ) + person1.file = File.create_from(io.BytesIO(sample_content), + content_type='text/plain', + extension='.txt') + self.assertIsInstance(person1.file, File) + self.assertEqual( + person1.file.locate(), + '%s/%s/%s?_ts=%s' % ( + self.base_url, prefix, person1.file.path, person1.file.timestamp ) + ) def test_default_base_url(self): store = S3Store( @@ -213,183 +207,177 @@ def test_default_base_url(self): assert store.base_url == 'https://%s.s3.amazonaws.com' % TEST_BUCKET def test_public_base_url_strip(self): - with mockup_s3_server(TEST_BUCKET) as (server, uri): - base_url = '%s/' % uri - StoreManager.register( - 's3', - functools.partial(create_s3_store, base_url=base_url), - default=True - ) + StoreManager.register( + 's3', + functools.partial(create_s3_store, base_url=self.base_url), + default=True + ) - class Person(self.Base): - __tablename__ = 'person' - id = Column(Integer, primary_key=True) - file = Column(File.as_mutable(Json)) + class Person(self.Base): + __tablename__ = 'person' + id = Column(Integer, primary_key=True) + file = Column(File.as_mutable(Json)) - session = self.create_all_and_get_session() + session = self.create_all_and_get_session() + person1 = Person() + self.assertIsNone(person1.file) + sample_content = b'Simple text.' + + with StoreManager(session): person1 = Person() - self.assertIsNone(person1.file) - sample_content = b'Simple text.' - - with StoreManager(session): - person1 = Person() - person1.file = File.create_from(io.BytesIO(sample_content), - content_type='text/plain', - extension='.txt') - self.assertIsInstance(person1.file, File) - self.assertEqual(person1.file.locate(), '%s%s?_ts=%s' % ( - base_url, person1.file.path, person1.file.timestamp)) + person1.file = File.create_from(io.BytesIO(sample_content), + content_type='text/plain', + extension='.txt') + self.assertIsInstance(person1.file, File) + self.assertEqual(person1.file.locate(), '%s/%s?_ts=%s' % ( + self.base_url, person1.file.path, person1.file.timestamp)) def test_cdn_url(self): cdn_url = 'http//test.sqlalchemy-media.com' - with mockup_s3_server(TEST_BUCKET) as (server, uri): - StoreManager.register( - 's3', - functools.partial( - create_s3_store, - base_url=uri, - cdn_url=cdn_url - ), - default=True - ) + StoreManager.register( + 's3', + functools.partial( + create_s3_store, + base_url=self.base_url, + cdn_url=cdn_url + ), + default=True + ) + + class Person(self.Base): + __tablename__ = 'person' + id = Column(Integer, primary_key=True) + file = Column(File.as_mutable(Json)) - class Person(self.Base): - __tablename__ = 'person' - id = Column(Integer, primary_key=True) - file = Column(File.as_mutable(Json)) + session = self.create_all_and_get_session() - session = self.create_all_and_get_session() + person1 = Person() + self.assertIsNone(person1.file) + sample_content = b'Simple text.' + with StoreManager(session): person1 = Person() - self.assertIsNone(person1.file) - sample_content = b'Simple text.' - - with StoreManager(session): - person1 = Person() - person1.file = File.create_from( - io.BytesIO(sample_content), - content_type='text/plain', - extension='.txt' - ) - self.assertIsInstance(person1.file, File) - self.assertEqual(person1.file.locate(), '%s/%s?_ts=%s' % ( - cdn_url, person1.file.path, person1.file.timestamp - )) + person1.file = File.create_from( + io.BytesIO(sample_content), + content_type='text/plain', + extension='.txt' + ) + self.assertIsInstance(person1.file, File) + self.assertEqual(person1.file.locate(), '%s/%s?_ts=%s' % ( + cdn_url, person1.file.path, person1.file.timestamp + )) def test_cdn_url_strip(self): cdn_url = 'http//test.sqlalchemy-media.com/' - with mockup_s3_server(TEST_BUCKET) as (server, uri): - StoreManager.register( - 's3', - functools.partial( - create_s3_store, - base_url=uri, - cdn_url=cdn_url - ), - default=True - ) + StoreManager.register( + 's3', + functools.partial( + create_s3_store, + base_url=self.base_url, + cdn_url=cdn_url + ), + default=True + ) + + class Person(self.Base): + __tablename__ = 'person' + id = Column(Integer, primary_key=True) + file = Column(File.as_mutable(Json)) - class Person(self.Base): - __tablename__ = 'person' - id = Column(Integer, primary_key=True) - file = Column(File.as_mutable(Json)) + session = self.create_all_and_get_session() - session = self.create_all_and_get_session() + person1 = Person() + self.assertIsNone(person1.file) + sample_content = b'Simple text.' + with StoreManager(session): person1 = Person() - self.assertIsNone(person1.file) - sample_content = b'Simple text.' - - with StoreManager(session): - person1 = Person() - person1.file = File.create_from( - io.BytesIO(sample_content), - content_type='text/plain', - extension='.txt' - ) - self.assertIsInstance(person1.file, File) - self.assertEqual(person1.file.locate(), '%s%s?_ts=%s' % ( - cdn_url, person1.file.path, person1.file.timestamp)) + person1.file = File.create_from( + io.BytesIO(sample_content), + content_type='text/plain', + extension='.txt' + ) + self.assertIsInstance(person1.file, File) + self.assertEqual(person1.file.locate(), '%s%s?_ts=%s' % ( + cdn_url, person1.file.path, person1.file.timestamp)) def test_cdn_url_with_prefix(self): prefix = 'media' cdn_url = 'http//test.sqlalchemy-media.com' - with mockup_s3_server(TEST_BUCKET) as (server, uri): - StoreManager.register( - 's3', - functools.partial( - create_s3_store, - prefix=prefix, - base_url=uri, - cdn_url=cdn_url - ), - default=True - ) + StoreManager.register( + 's3', + functools.partial( + create_s3_store, + prefix=prefix, + base_url=self.base_url, + cdn_url=cdn_url + ), + default=True + ) + + class Person(self.Base): + __tablename__ = 'person' + id = Column(Integer, primary_key=True) + file = Column(File.as_mutable(Json)) - class Person(self.Base): - __tablename__ = 'person' - id = Column(Integer, primary_key=True) - file = Column(File.as_mutable(Json)) + session = self.create_all_and_get_session() - session = self.create_all_and_get_session() + person1 = Person() + self.assertIsNone(person1.file) + sample_content = b'Simple text.' + with StoreManager(session): person1 = Person() - self.assertIsNone(person1.file) - sample_content = b'Simple text.' - - with StoreManager(session): - person1 = Person() - person1.file = File.create_from(io.BytesIO(sample_content), - content_type='text/plain', - extension='.txt') - self.assertIsInstance(person1.file, File) - self.assertEqual(person1.file.locate(), '%s/%s/%s?_ts=%s' % ( - cdn_url, prefix, person1.file.path, person1.file.timestamp) - ) + person1.file = File.create_from(io.BytesIO(sample_content), + content_type='text/plain', + extension='.txt') + self.assertIsInstance(person1.file, File) + self.assertEqual(person1.file.locate(), '%s/%s/%s?_ts=%s' % ( + cdn_url, prefix, person1.file.path, person1.file.timestamp) + ) def test_cdn_url_with_ignore_prefix(self): prefix = 'media' cdn_url = 'http//test.sqlalchemy-media.com' - with mockup_s3_server(TEST_BUCKET) as (server, uri): - StoreManager.register( - 's3', - functools.partial( - create_s3_store, - prefix=prefix, - base_url=uri, - cdn_url=cdn_url, - cdn_prefix_ignore=True - ), - default=True - ) + StoreManager.register( + 's3', + functools.partial( + create_s3_store, + prefix=prefix, + base_url=self.base_url, + cdn_url=cdn_url, + cdn_prefix_ignore=True + ), + default=True + ) + + class Person(self.Base): + __tablename__ = 'person' + id = Column(Integer, primary_key=True) + file = Column(File.as_mutable(Json)) - class Person(self.Base): - __tablename__ = 'person' - id = Column(Integer, primary_key=True) - file = Column(File.as_mutable(Json)) + session = self.create_all_and_get_session() - session = self.create_all_and_get_session() + person1 = Person() + self.assertIsNone(person1.file) + sample_content = b'Simple text.' + with StoreManager(session): person1 = Person() - self.assertIsNone(person1.file) - sample_content = b'Simple text.' - - with StoreManager(session): - person1 = Person() - person1.file = File.create_from( - io.BytesIO(sample_content), - content_type='text/plain', - extension='.txt' - ) - self.assertIsInstance(person1.file, File) - self.assertEqual( - person1.file.locate(), '%s/%s?_ts=%s' % ( - cdn_url, - person1.file.path, - person1.file.timestamp - ) + person1.file = File.create_from( + io.BytesIO(sample_content), + content_type='text/plain', + extension='.txt' + ) + self.assertIsInstance(person1.file, File) + self.assertEqual( + person1.file.locate(), '%s/%s?_ts=%s' % ( + cdn_url, + person1.file.path, + person1.file.timestamp ) + ) if __name__ == '__main__': # pragma: no cover From 02e70c9c3c6a1d32aa982e0ef6cccad584ccfc75 Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 22 Aug 2021 17:57:57 +0300 Subject: [PATCH 10/20] Change the value for base_url property for ssh test cases --- sqlalchemy_media/tests/test_ssh_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlalchemy_media/tests/test_ssh_store.py b/sqlalchemy_media/tests/test_ssh_store.py index cf4dc1d..191e72a 100644 --- a/sqlalchemy_media/tests/test_ssh_store.py +++ b/sqlalchemy_media/tests/test_ssh_store.py @@ -12,7 +12,7 @@ class SSHStoreTestCase(MockupSSHTestCase): def setUp(self): super().setUp() - self.base_url = 'http://static1.example.orm' + self.base_url = 'http://localhost:9000' self.stuff_path = join(self.here, 'stuff') self.sample_text_file1 = join(self.stuff_path, 'sample_text_file1.txt') From d0bfc86d28225dc900e4e66fab33ac7197432dc8 Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 22 Aug 2021 18:09:21 +0300 Subject: [PATCH 11/20] Fix the test in ImageTestCase by adding parameterization to the alchemy session factory --- sqlalchemy_media/tests/helpers/testcases.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlalchemy_media/tests/helpers/testcases.py b/sqlalchemy_media/tests/helpers/testcases.py index 1365c57..dd3f4b9 100644 --- a/sqlalchemy_media/tests/helpers/testcases.py +++ b/sqlalchemy_media/tests/helpers/testcases.py @@ -20,13 +20,13 @@ def setUp(self): self.Base = declarative_base() self.engine = create_engine(self.db_uri, echo=False) - def create_all_and_get_session(self): + def create_all_and_get_session(self, **options): self.Base.metadata.create_all(self.engine, checkfirst=True) self.session_factory = sessionmaker( bind=self.engine, autoflush=False, autocommit=False, - expire_on_commit=True, + expire_on_commit=options.get('expire_on_commit', False), twophase=False ) return self.session_factory() From cabc33ff061c3d3bbe945cc5a2dc7dc57d12d02c Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 5 Sep 2021 12:09:09 +0300 Subject: [PATCH 12/20] Replace pypi with git source for requests-aws4auth --- requirements-optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-optional.txt b/requirements-optional.txt index f39c637..3cf06c2 100644 --- a/requirements-optional.txt +++ b/requirements-optional.txt @@ -1,5 +1,5 @@ httpx paramiko -requests-aws4auth >= 0.9 +git+https://github.com/tedder/requests-aws4auth.git@master requests-aliyun >= 0.2.5 Werkzeug \ No newline at end of file From d1ab4eda20ff621b9b593611c48dd6174ad63922 Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 5 Sep 2021 12:29:22 +0300 Subject: [PATCH 13/20] Move config constants to a separate module --- sqlalchemy_media/tests/helpers/config.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 sqlalchemy_media/tests/helpers/config.py diff --git a/sqlalchemy_media/tests/helpers/config.py b/sqlalchemy_media/tests/helpers/config.py new file mode 100644 index 0000000..a88d5c7 --- /dev/null +++ b/sqlalchemy_media/tests/helpers/config.py @@ -0,0 +1,7 @@ + +# Settings for S3 storage + +TEST_BUCKET = 'test-bucket' +TEST_ACCESS_KEY = 'test_access_key' +TEST_SECRET_KEY = 'test_secret_key' +TEST_REGION = 'ap-northeast-2' From 9fb54ea5ab780631ed81976391c7b1599451fe7e Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 5 Sep 2021 13:21:50 +0300 Subject: [PATCH 14/20] Add create_s3_store to s3 helpers --- sqlalchemy_media/tests/helpers/s3.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/sqlalchemy_media/tests/helpers/s3.py b/sqlalchemy_media/tests/helpers/s3.py index beca23d..f2efd2f 100644 --- a/sqlalchemy_media/tests/helpers/s3.py +++ b/sqlalchemy_media/tests/helpers/s3.py @@ -3,6 +3,9 @@ import httpx +from sqlalchemy_media.stores import S3Store + +from .config import TEST_ACCESS_KEY, TEST_BUCKET, TEST_REGION, TEST_SECRET_KEY from .http import simple_http_server @@ -13,8 +16,8 @@ def mockup_s3_server(bucket, **kwargs): mock_app.debug = False with simple_http_server( WSGIRequestHandler, - server_class=WSGIServer, - app=mock_app, + server_class=WSGIServer, + app=mock_app, **kwargs ) as server: url = 'http://localhost:%s' % server.server_address[1] @@ -23,3 +26,13 @@ def mockup_s3_server(bucket, **kwargs): res = httpx.put(bucket_uri) assert res.status_code == 200 yield server, bucket_uri + + +def create_s3_store(bucket=TEST_BUCKET, **kwargs): + return S3Store( + bucket, + TEST_ACCESS_KEY, + TEST_SECRET_KEY, + TEST_REGION, + **kwargs + ) From 1ce17d167be6bb7195be3a5e71725bc118cd1aba Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 5 Sep 2021 13:36:54 +0300 Subject: [PATCH 15/20] Add S3TestCase. Replace options with keyword arguments --- sqlalchemy_media/tests/helpers/testcases.py | 25 +++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/sqlalchemy_media/tests/helpers/testcases.py b/sqlalchemy_media/tests/helpers/testcases.py index dd3f4b9..e74fe40 100644 --- a/sqlalchemy_media/tests/helpers/testcases.py +++ b/sqlalchemy_media/tests/helpers/testcases.py @@ -10,6 +10,9 @@ from sqlalchemy_media import StoreManager, FileSystemStore +from .config import TEST_BUCKET +from .s3 import mockup_s3_server, create_s3_store + class SqlAlchemyTestCase(unittest.TestCase): @classmethod @@ -20,18 +23,36 @@ def setUp(self): self.Base = declarative_base() self.engine = create_engine(self.db_uri, echo=False) - def create_all_and_get_session(self, **options): + def create_all_and_get_session(self, expire_on_commit: bool = False): + """ + A factory method for making a SQLAlchemy session factory object. + :param expire_on_commit: @see: https://docs.sqlalchemy.org/en/14/orm/session_api.html?highlight=expire_on_commit#sqlalchemy.orm.Session.params.expire_on_commit + """ self.Base.metadata.create_all(self.engine, checkfirst=True) self.session_factory = sessionmaker( bind=self.engine, autoflush=False, autocommit=False, - expire_on_commit=options.get('expire_on_commit', False), + expire_on_commit=expire_on_commit, twophase=False ) return self.session_factory() +class S3TestCase(unittest.TestCase): + """Mixin for runnning the S3 server""" + + def run(self, result): + with mockup_s3_server(bucket=TEST_BUCKET) as (server, bucket_uri): + self.storage = create_s3_store( + bucket=TEST_BUCKET, base_url=bucket_uri + ) + self.bucket_name = TEST_BUCKET + self.server = server + self.base_url = bucket_uri + super(S3TestCase, self).run(result) + + class TempStoreTestCase(SqlAlchemyTestCase): @classmethod def setUpClass(cls): From 604739f7b2e5803c3ef59492b91bb35e9a6aec65 Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 5 Sep 2021 14:46:50 +0300 Subject: [PATCH 16/20] Move S3TestCase to s3 helpers --- sqlalchemy_media/tests/helpers/testcases.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/sqlalchemy_media/tests/helpers/testcases.py b/sqlalchemy_media/tests/helpers/testcases.py index e74fe40..ed5a9eb 100644 --- a/sqlalchemy_media/tests/helpers/testcases.py +++ b/sqlalchemy_media/tests/helpers/testcases.py @@ -39,20 +39,6 @@ def create_all_and_get_session(self, expire_on_commit: bool = False): return self.session_factory() -class S3TestCase(unittest.TestCase): - """Mixin for runnning the S3 server""" - - def run(self, result): - with mockup_s3_server(bucket=TEST_BUCKET) as (server, bucket_uri): - self.storage = create_s3_store( - bucket=TEST_BUCKET, base_url=bucket_uri - ) - self.bucket_name = TEST_BUCKET - self.server = server - self.base_url = bucket_uri - super(S3TestCase, self).run(result) - - class TempStoreTestCase(SqlAlchemyTestCase): @classmethod def setUpClass(cls): From d9bb8b7da270334bd4b4580b33f709cec2dca7a2 Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 5 Sep 2021 14:47:21 +0300 Subject: [PATCH 17/20] Add S3TestCase to s3 helpers --- sqlalchemy_media/tests/helpers/s3.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sqlalchemy_media/tests/helpers/s3.py b/sqlalchemy_media/tests/helpers/s3.py index f2efd2f..f04022a 100644 --- a/sqlalchemy_media/tests/helpers/s3.py +++ b/sqlalchemy_media/tests/helpers/s3.py @@ -1,3 +1,4 @@ +import unittest import contextlib from wsgiref.simple_server import WSGIRequestHandler, WSGIServer @@ -36,3 +37,17 @@ def create_s3_store(bucket=TEST_BUCKET, **kwargs): TEST_REGION, **kwargs ) + + +class S3TestCase(unittest.TestCase): + """Mixin for runnning the S3 server""" + + def run(self, result): + with mockup_s3_server(bucket=TEST_BUCKET) as (server, bucket_uri): + self.storage = create_s3_store( + bucket=TEST_BUCKET, base_url=bucket_uri + ) + self.bucket_name = TEST_BUCKET + self.server = server + self.base_url = bucket_uri + super(S3TestCase, self).run(result) From f9bfc744e173b7efa86b00e9279602326b69750e Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 5 Sep 2021 14:50:47 +0300 Subject: [PATCH 18/20] Add imports from config and s3 helpers --- sqlalchemy_media/tests/helpers/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlalchemy_media/tests/helpers/__init__.py b/sqlalchemy_media/tests/helpers/__init__.py index 244d202..68d5978 100644 --- a/sqlalchemy_media/tests/helpers/__init__.py +++ b/sqlalchemy_media/tests/helpers/__init__.py @@ -1,7 +1,7 @@ - +from .config import TEST_ACCESS_KEY, TEST_BUCKET, TEST_REGION, TEST_SECRET_KEY from .http import simple_http_server, encode_multipart_data from .os2 import mockup_os2_server -from .s3 import mockup_s3_server +from .s3 import mockup_s3_server, create_s3_store, S3TestCase from .ssh import MockupSSHServer, MockupSSHTestCase from .ftp import MockFTP from .static import mockup_http_static_server From 828603ecbc380c3e6e1d7a606d368da175f8b88e Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 5 Sep 2021 15:15:27 +0300 Subject: [PATCH 19/20] Reffactor tests for s3 storage --- sqlalchemy_media/tests/test_s3_store.py | 64 ++++++++----------------- 1 file changed, 21 insertions(+), 43 deletions(-) diff --git a/sqlalchemy_media/tests/test_s3_store.py b/sqlalchemy_media/tests/test_s3_store.py index 7a0da9c..96e685b 100644 --- a/sqlalchemy_media/tests/test_s3_store.py +++ b/sqlalchemy_media/tests/test_s3_store.py @@ -7,30 +7,18 @@ from sqlalchemy_media.attachments import File, Image as BaseImage, \ Thumbnail as BaseThumbnail +from sqlalchemy_media.stores import StoreManager, S3Store from sqlalchemy_media.exceptions import S3Error -from sqlalchemy_media.stores import S3Store -from sqlalchemy_media.stores import StoreManager -from sqlalchemy_media.tests.helpers import Json, SqlAlchemyTestCase, \ - mockup_s3_server +from sqlalchemy_media.tests.helpers import ( + TEST_BUCKET, + Json, + S3TestCase, + SqlAlchemyTestCase, + create_s3_store, +) -TEST_BUCKET = 'test-bucket' -TEST_ACCESS_KEY = 'test_access_key' -TEST_SECRET_KEY = 'test_secret_key' -TEST_REGION = 'ap-northeast-2' - - -def create_s3_store(bucket=TEST_BUCKET, **kwargs): - return S3Store( - bucket, - TEST_ACCESS_KEY, - TEST_SECRET_KEY, - TEST_REGION, - **kwargs - ) - - -class S3StoreTestCase(SqlAlchemyTestCase): +class S3StoreTestCase(SqlAlchemyTestCase, S3TestCase): @classmethod def setUpClass(cls): @@ -38,17 +26,15 @@ def setUpClass(cls): cls.this_dir = abspath(dirname(__file__)) cls.stuff_path = join(cls.this_dir, 'stuff') cls.dog_jpeg = join(cls.stuff_path, 'dog.jpg') - cls.base_url = 'http://localhost:9000/test-bucket' cls.sample_text_file1 = join(cls.stuff_path, 'sample_text_file1.txt') def test_put_from_stream(self): - store = create_s3_store(base_url=self.base_url) target_filename = 'test_put_from_stream/file_from_stream1.txt' content = b'Lorem ipsum dolor sit amet' stream = io.BytesIO(content) - length = store.put(target_filename, stream) + length = self.storage.put(target_filename, stream) self.assertEqual(length, len(content)) - self.assertIsInstance(store.open(target_filename), io.BytesIO) + self.assertIsInstance(self.storage.open(target_filename), io.BytesIO) def test_put_error(self): store = create_s3_store(base_url=self.base_url[:-2]) @@ -95,40 +81,37 @@ class Person(self.Base): self.assertTrue(thumbnail.reproducible, True) def test_delete(self): - store = create_s3_store(base_url=self.base_url) target_filename = 'test_delete/sample_text_file1.txt' with open(self.sample_text_file1, 'rb') as f: - length = store.put(target_filename, f) + length = self.storage.put(target_filename, f) self.assertEqual(length, getsize(self.sample_text_file1)) - self.assertIsInstance(store.open(target_filename), io.BytesIO) + self.assertIsInstance(self.storage.open(target_filename), io.BytesIO) - store.delete(target_filename) + self.storage.delete(target_filename) with self.assertRaises(S3Error): - store.open(target_filename) + self.storage.open(target_filename) def test_delete_error(self): - store = create_s3_store(base_url=self.base_url) wrong_store = create_s3_store(base_url=self.base_url[:-2]) target_filename = 'test_delete/sample_text_file1.txt' with open(self.sample_text_file1, 'rb') as f: - length = store.put(target_filename, f) + length = self.storage.put(target_filename, f) self.assertEqual(length, getsize(self.sample_text_file1)) - self.assertIsInstance(store.open(target_filename), io.BytesIO) + self.assertIsInstance(self.storage.open(target_filename), io.BytesIO) with self.assertRaises(S3Error): wrong_store.delete(target_filename) def test_open(self): - store = create_s3_store(base_url=self.base_url) target_filename = 'test_delete/sample_text_file1.txt' with open(self.sample_text_file1, 'rb') as f: - length = store.put(target_filename, f) + length = self.storage.put(target_filename, f) self.assertEqual(length, getsize(self.sample_text_file1)) - self.assertIsInstance(store.open(target_filename), io.BytesIO) + self.assertIsInstance(self.storage.open(target_filename), io.BytesIO) # Reading - with store.open(target_filename, mode='rb') as stored_file, \ + with self.storage.open(target_filename, mode='rb') as stored_file, \ open(self.sample_text_file1, mode='rb') as original_file: self.assertEqual(stored_file.read(), original_file.read()) @@ -198,12 +181,7 @@ class Person(self.Base): ) def test_default_base_url(self): - store = S3Store( - TEST_BUCKET, - TEST_ACCESS_KEY, - TEST_SECRET_KEY, - TEST_REGION - ) + store = create_s3_store() assert store.base_url == 'https://%s.s3.amazonaws.com' % TEST_BUCKET def test_public_base_url_strip(self): From 5b125dbf7ab0c8f2348bce95a493b3f1abfd8242 Mon Sep 17 00:00:00 2001 From: TheLazzziest Date: Sun, 5 Sep 2021 15:24:31 +0300 Subject: [PATCH 20/20] Remove docker-compose config --- docker-compose.yml | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 1654681..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: "3.8" - -services: - storage: - image: minio/minio:latest - entrypoint: sh - command: -c "mkdir -p /data/test-bucket && minio server /data --console-address \":9001\"" - volumes: - - ./data:/data - ports: - - 9000:9000 - - 9001:9001 - environment: - - MINIO_ROOT_USER=test_access_key - - MINIO_ROOT_PASSWORD=test_secret_key - - MINIO_REGION_NAME=ap-northeast-2