From 8108360651b53871563617e0747fb3345871a51f Mon Sep 17 00:00:00 2001 From: Simon Ratcliffe Date: Wed, 24 Apr 2019 15:41:12 +0200 Subject: [PATCH 1/7] Included tests for bucket expiry. We can't test expiry directly, firstly because minio doesn't support lifecycle polices, and secondly because the minimum expiry time is 1 day. This basically just test the infrastructure and that the schema for the policy is created correctly. --- katdal/chunkstore.py | 3 ++ katdal/chunkstore_s3.py | 9 +++-- katdal/schemas/__init__.py | 39 +++++++++++++++++++++ katdal/schemas/minimal_lifecycle_policy.xsd | 38 ++++++++++++++++++++ katdal/test/test_chunkstore_s3.py | 21 ++++++++++- 5 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 katdal/schemas/__init__.py create mode 100644 katdal/schemas/minimal_lifecycle_policy.xsd diff --git a/katdal/chunkstore.py b/katdal/chunkstore.py index fb35372c..7bc78d09 100644 --- a/katdal/chunkstore.py +++ b/katdal/chunkstore.py @@ -37,6 +37,9 @@ class ChunkStoreError(Exception): class StoreUnavailable(OSError, ChunkStoreError): """Could not access underlying storage medium (offline, auth failed, etc).""" +class NotSupported(ChunkStoreError): + """The underlying store does not support the requested operation + (e.g. minio doesn't support lifecycle policies on buckets)""" class ChunkNotFound(KeyError, ChunkStoreError): """The store was accessible but a chunk with the given name was not found.""" diff --git a/katdal/chunkstore_s3.py b/katdal/chunkstore_s3.py index 4555c94f..78d26f21 100644 --- a/katdal/chunkstore_s3.py +++ b/katdal/chunkstore_s3.py @@ -24,7 +24,7 @@ standard_library.install_aliases() # noqa: E402 from builtins import object import future.utils -from future.utils import raise_, bytes_to_native_str +from future.utils import raise_, bytes_to_native_str, raise_from import contextlib import io @@ -53,9 +53,11 @@ botocore = None from .chunkstore import (ChunkStore, StoreUnavailable, ChunkNotFound, BadChunk, - npy_header_and_body) + NotSupported, npy_header_and_body) from .sensordata import to_str +from . import schemas + # Lifecycle policies unfortunately use XML encoding rather than JSON # Following path of least resistance we simply .format() this string @@ -180,6 +182,8 @@ def _raise_for_status(response): except requests.HTTPError as error: if response.status_code == 404: raise ChunkNotFound(str(error)) + elif response.status_code == 501: + raise_from(NotSupported(str(error)), error) else: raise StoreUnavailable(str(error)) @@ -467,6 +471,7 @@ def create_array(self, array_name): if self.expiry_days > 0: xml_payload = _BASE_LIFECYCLE_POLICY.format(self.expiry_days) + schemas.MINIMAL_LIFECYCLE_POLICY.validate(xml_payload) b64_md5 = base64.b64encode(hashlib.md5(xml_payload.encode('utf-8')).digest()).decode('utf-8') lifecycle_headers = {'Content-Type': 'text/xml', 'Content-MD5': b64_md5} with self.request(None, 'PUT', url, params='lifecycle', data=xml_payload, headers=lifecycle_headers): diff --git a/katdal/schemas/__init__.py b/katdal/schemas/__init__.py new file mode 100644 index 00000000..710ea1ad --- /dev/null +++ b/katdal/schemas/__init__.py @@ -0,0 +1,39 @@ +"""Makes packaged XSD schemas available as validators.""" + +from lxml import etree + +import pkg_resources + +#def _make_validator(schema): +# """Check a schema document and create a validator from it""" +# validator_cls = jsonschema.validators.validator_for(schema) +# validator_cls.check_schema(schema) +# return validator_cls(schema, format_checker=jsonschema.FormatChecker()) + + +class ValidatorWithLog(object): + def __init__(self, validator): + self.validator = validator + + def validate(self, xml_string): + """Validates a supplied XML string against + the instantiated validator. + Failure to validate will raise DocumentInvalid + with the output of the error log.""" + try: + xml_doc = etree.fromstring(bytes(bytearray(xml_string, encoding='utf-8'))) + except etree.XMLSyntaxError as e: + raise ValueError(e) + if not self.validator.validate(xml_doc): + log = self.validator.error_log + raise etree.DocumentInvalid(log.last_error) + return True + +for name in pkg_resources.resource_listdir(__name__, '.'): + if name.endswith('.xsd'): + #with open(pkg_resources.resource_stream(__name__, name)) as f: + xmlschema_doc = etree.parse(pkg_resources.resource_stream(__name__, name)) + xml_validator = etree.XMLSchema(xmlschema_doc) + #reader = codecs.getreader('utf-8')(pkg_resources.resource_stream(__name__, name)) + #schema = json.load(reader) + globals()[name[:-4].upper()] = ValidatorWithLog(xml_validator) #_make_validator(schema) diff --git a/katdal/schemas/minimal_lifecycle_policy.xsd b/katdal/schemas/minimal_lifecycle_policy.xsd new file mode 100644 index 00000000..87feaea5 --- /dev/null +++ b/katdal/schemas/minimal_lifecycle_policy.xsd @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/katdal/test/test_chunkstore_s3.py b/katdal/test/test_chunkstore_s3.py index b914642b..7cddf687 100644 --- a/katdal/test/test_chunkstore_s3.py +++ b/katdal/test/test_chunkstore_s3.py @@ -51,11 +51,17 @@ from nose.tools import assert_raises, assert_equal, timed import mock import requests +import lxml from katdal.chunkstore_s3 import S3ChunkStore, _AWSAuth, read_array -from katdal.chunkstore import StoreUnavailable +from katdal.chunkstore import StoreUnavailable, NotSupported from katdal.test.test_chunkstore import ChunkStoreTestBase +# No expiration rule included +_INVALID_LIFECYCLE_POLICY = """ + +katdal_expiry_{0}_daysEnabled +""" def gethostbyname_slow(host): """Mock DNS lookup that is meant to be slow.""" @@ -231,6 +237,19 @@ def test_public_read(self): y = reader.get_chunk('public', slices, x.dtype) np.testing.assert_array_equal(x, y) + def test_bucket_expiry(self): + # NOTE: Minimum bucket expiry time is 1 day so real world testing is impractical. + # First we check the default policy is at least valid, even though minio + # doesn't actually support setting it. + test_store = self.from_url(self.url, expiry_days=1) + assert_raises(NotSupported, test_store.create_array, 'test-expiry') + + @mock.patch('katdal.chunkstore_s3._BASE_LIFECYCLE_POLICY', _INVALID_LIFECYCLE_POLICY) + def test_bucket_expiry_invalid(self): + # now test with an invalid policy + test_store = self.from_url(self.url, expiry_days=1) + assert_raises(lxml.etree.DocumentInvalid, test_store.create_array, 'test-expiry') + @timed(0.1 + 0.05) def test_store_unavailable_invalid_url(self): # Ensure that timeouts work From 156309fadb0475d757a64df95ec46525b5931084 Mon Sep 17 00:00:00 2001 From: Simon Ratcliffe Date: Wed, 24 Apr 2019 15:53:02 +0200 Subject: [PATCH 2/7] Flake8 and docstrings --- katdal/chunkstore.py | 2 ++ katdal/schemas/__init__.py | 31 +++++++++++++++++-------------- katdal/test/test_chunkstore_s3.py | 11 +++++++++-- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/katdal/chunkstore.py b/katdal/chunkstore.py index 7bc78d09..72080b80 100644 --- a/katdal/chunkstore.py +++ b/katdal/chunkstore.py @@ -37,10 +37,12 @@ class ChunkStoreError(Exception): class StoreUnavailable(OSError, ChunkStoreError): """Could not access underlying storage medium (offline, auth failed, etc).""" + class NotSupported(ChunkStoreError): """The underlying store does not support the requested operation (e.g. minio doesn't support lifecycle policies on buckets)""" + class ChunkNotFound(KeyError, ChunkStoreError): """The store was accessible but a chunk with the given name was not found.""" diff --git a/katdal/schemas/__init__.py b/katdal/schemas/__init__.py index 710ea1ad..3fcaccdd 100644 --- a/katdal/schemas/__init__.py +++ b/katdal/schemas/__init__.py @@ -4,22 +4,27 @@ import pkg_resources -#def _make_validator(schema): -# """Check a schema document and create a validator from it""" -# validator_cls = jsonschema.validators.validator_for(schema) -# validator_cls.check_schema(schema) -# return validator_cls(schema, format_checker=jsonschema.FormatChecker()) - class ValidatorWithLog(object): def __init__(self, validator): self.validator = validator def validate(self, xml_string): - """Validates a supplied XML string against - the instantiated validator. - Failure to validate will raise DocumentInvalid - with the output of the error log.""" + """Validates a supplied XML string against the instantiated validator. + + Parameters + --------- + xml_string : str + String representation of the XML to be turned into a document + and validated. + + Raises + ------ + etree.DocumentInvalid + if `xml_string` does not validate against the XSD schema + ValueError + if `xml_string` cannot be parsed into a valid XML document + """ try: xml_doc = etree.fromstring(bytes(bytearray(xml_string, encoding='utf-8'))) except etree.XMLSyntaxError as e: @@ -29,11 +34,9 @@ def validate(self, xml_string): raise etree.DocumentInvalid(log.last_error) return True + for name in pkg_resources.resource_listdir(__name__, '.'): if name.endswith('.xsd'): - #with open(pkg_resources.resource_stream(__name__, name)) as f: xmlschema_doc = etree.parse(pkg_resources.resource_stream(__name__, name)) xml_validator = etree.XMLSchema(xmlschema_doc) - #reader = codecs.getreader('utf-8')(pkg_resources.resource_stream(__name__, name)) - #schema = json.load(reader) - globals()[name[:-4].upper()] = ValidatorWithLog(xml_validator) #_make_validator(schema) + globals()[name[:-4].upper()] = ValidatorWithLog(xml_validator) diff --git a/katdal/test/test_chunkstore_s3.py b/katdal/test/test_chunkstore_s3.py index 7cddf687..796205ab 100644 --- a/katdal/test/test_chunkstore_s3.py +++ b/katdal/test/test_chunkstore_s3.py @@ -63,6 +63,7 @@ katdal_expiry_{0}_daysEnabled """ + def gethostbyname_slow(host): """Mock DNS lookup that is meant to be slow.""" time.sleep(30) @@ -245,11 +246,17 @@ def test_bucket_expiry(self): assert_raises(NotSupported, test_store.create_array, 'test-expiry') @mock.patch('katdal.chunkstore_s3._BASE_LIFECYCLE_POLICY', _INVALID_LIFECYCLE_POLICY) - def test_bucket_expiry_invalid(self): - # now test with an invalid policy + def test_bucket_expiry_invalid_schema(self): + # Now test with an invalid policy test_store = self.from_url(self.url, expiry_days=1) assert_raises(lxml.etree.DocumentInvalid, test_store.create_array, 'test-expiry') + @mock.patch('katdal.chunkstore_s3._BASE_LIFECYCLE_POLICY', "") + def test_bucket_expiry_not_xml(self): + # Code path coverage to test a policy that is not even valid XML + test_store = self.from_url(self.url, expiry_days=1) + assert_raises(ValueError, test_store.create_array, 'test-expiry') + @timed(0.1 + 0.05) def test_store_unavailable_invalid_url(self): # Ensure that timeouts work From 44cb6e9e6fcd28d009b3355c88607ab99b137473 Mon Sep 17 00:00:00 2001 From: Simon Ratcliffe Date: Wed, 24 Apr 2019 16:59:20 +0200 Subject: [PATCH 3/7] Allow switchable validation to avoid hard lxml dependency --- katdal/chunkstore_s3.py | 31 +++++++++++++++++++++++-------- katdal/schemas/__init__.py | 13 ++++++++++--- katdal/test/test_chunkstore_s3.py | 13 ++++++++----- test-requirements.txt | 1 + 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/katdal/chunkstore_s3.py b/katdal/chunkstore_s3.py index 78d26f21..6faa7a28 100644 --- a/katdal/chunkstore_s3.py +++ b/katdal/chunkstore_s3.py @@ -284,6 +284,11 @@ class S3ChunkStore(ChunkStore): expiry_days : int If set to a value greater than 0 will set a future expiry time in days for any new buckets created. + validate_xml_policies : bool + If set to true, S3 operations that use XML policies will be validated + against the inbuilt schemas. Note that these are relatively minimal + and not a guarantee of operation success on passing validation. + Requires lxml Raises @@ -292,7 +297,7 @@ class S3ChunkStore(ChunkStore): If requests is not installed (it's an optional dependency otherwise) """ - def __init__(self, session_factory, url, public_read=False, expiry_days=0): + def __init__(self, session_factory, url, public_read=False, expiry_days=0, validate_xml_policies=False): try: # Quick smoke test to see if the S3 server is available, by listing # buckets. Depending on the server in use, this may return a 403 @@ -313,9 +318,10 @@ def __init__(self, session_factory, url, public_read=False, expiry_days=0): self._url = to_str(url) self.public_read = public_read self.expiry_days = int(expiry_days) + self.validate_xml_policies = validate_xml_policies @classmethod - def _from_url(cls, url, timeout, token, credentials, public_read, expiry_days): + def _from_url(cls, url, timeout, token, credentials, public_read, expiry_days, validate_xml_policies): """Construct S3 chunk store from endpoint URL (see :meth:`from_url`).""" if token is not None: parsed = urllib.parse.urlparse(url) @@ -337,12 +343,12 @@ def session_factory(): session.mount(url, adapter) return session - return cls(session_factory, url, public_read, expiry_days) + return cls(session_factory, url, public_read, expiry_days, validate_xml_policies) @classmethod def from_url(cls, url, timeout=300, extra_timeout=10, token=None, credentials=None, public_read=False, - expiry_days=0, **kwargs): + expiry_days=0, validate_xml_policies=False, **kwargs): """Construct S3 chunk store from endpoint URL. Parameters @@ -364,6 +370,11 @@ def from_url(cls, url, timeout=300, extra_timeout=10, expiry_days : int If set to a value greater than 0 will set a future expiry time in days for any new buckets created. + validate_xml_policies : bool + If set to true, S3 operations that use XML policies will be validated + against the inbuilt schemas. Note that these are relatively minimal + and not a guarantee of operation success on passing validation. + Requires lxml kwargs : dict Extra keyword arguments (unused) @@ -379,15 +390,16 @@ def from_url(cls, url, timeout=300, extra_timeout=10, # (avoiding extra dependency on Python 2, revisit when Python 3 only) q = queue.Queue() - def _from_url(url, timeout, token, credentials, public_read, expiry_days): + def _from_url(url, timeout, token, credentials, public_read, expiry_days, validate_xml_policies): """Construct chunk store and return it (or exception) via queue.""" try: - q.put(cls._from_url(url, timeout, token, credentials, public_read, expiry_days)) + q.put(cls._from_url(url, timeout, token, credentials, public_read, expiry_days, validate_xml_policies)) except BaseException: q.put(sys.exc_info()) thread = threading.Thread(target=_from_url, - args=(url, timeout, token, credentials, public_read, expiry_days)) + args=(url, timeout, token, credentials, public_read, expiry_days, + validate_xml_policies)) thread.daemon = True thread.start() if timeout is not None: @@ -471,7 +483,10 @@ def create_array(self, array_name): if self.expiry_days > 0: xml_payload = _BASE_LIFECYCLE_POLICY.format(self.expiry_days) - schemas.MINIMAL_LIFECYCLE_POLICY.validate(xml_payload) + if self.validate_xml_policies: + if not schemas.has_lxml: + raise ImportError("XML schema validation requires lxml to be installed.") + schemas.MINIMAL_LIFECYCLE_POLICY.validate(xml_payload) b64_md5 = base64.b64encode(hashlib.md5(xml_payload.encode('utf-8')).digest()).decode('utf-8') lifecycle_headers = {'Content-Type': 'text/xml', 'Content-MD5': b64_md5} with self.request(None, 'PUT', url, params='lifecycle', data=xml_payload, headers=lifecycle_headers): diff --git a/katdal/schemas/__init__.py b/katdal/schemas/__init__.py index 3fcaccdd..c3cb9fcd 100644 --- a/katdal/schemas/__init__.py +++ b/katdal/schemas/__init__.py @@ -1,10 +1,17 @@ """Makes packaged XSD schemas available as validators.""" -from lxml import etree - import pkg_resources +has_lxml = False + +try: + from lxml import etree + has_lxml = True +except ImportError: + pass + + class ValidatorWithLog(object): def __init__(self, validator): self.validator = validator @@ -36,7 +43,7 @@ def validate(self, xml_string): for name in pkg_resources.resource_listdir(__name__, '.'): - if name.endswith('.xsd'): + if name.endswith('.xsd') and has_lxml: xmlschema_doc = etree.parse(pkg_resources.resource_stream(__name__, name)) xml_validator = etree.XMLSchema(xmlschema_doc) globals()[name[:-4].upper()] = ValidatorWithLog(xml_validator) diff --git a/katdal/test/test_chunkstore_s3.py b/katdal/test/test_chunkstore_s3.py index 796205ab..8fa72f7d 100644 --- a/katdal/test/test_chunkstore_s3.py +++ b/katdal/test/test_chunkstore_s3.py @@ -45,13 +45,13 @@ import contextlib import io import warnings +import lxml import numpy as np from nose import SkipTest from nose.tools import assert_raises, assert_equal, timed import mock import requests -import lxml from katdal.chunkstore_s3 import S3ChunkStore, _AWSAuth, read_array from katdal.chunkstore import StoreUnavailable, NotSupported @@ -240,21 +240,24 @@ def test_public_read(self): def test_bucket_expiry(self): # NOTE: Minimum bucket expiry time is 1 day so real world testing is impractical. - # First we check the default policy is at least valid, even though minio - # doesn't actually support setting it. + # We expect not supported since minio doesn't allow lifecycle policies test_store = self.from_url(self.url, expiry_days=1) assert_raises(NotSupported, test_store.create_array, 'test-expiry') + def test_bucket_expiry_with_validation(self): + test_store = self.from_url(self.url, expiry_days=1, validate_xml_policies=True) + assert_raises(NotSupported, test_store.create_array, 'test-expiry') + @mock.patch('katdal.chunkstore_s3._BASE_LIFECYCLE_POLICY', _INVALID_LIFECYCLE_POLICY) def test_bucket_expiry_invalid_schema(self): # Now test with an invalid policy - test_store = self.from_url(self.url, expiry_days=1) + test_store = self.from_url(self.url, expiry_days=1, validate_xml_policies=True) assert_raises(lxml.etree.DocumentInvalid, test_store.create_array, 'test-expiry') @mock.patch('katdal.chunkstore_s3._BASE_LIFECYCLE_POLICY', "") def test_bucket_expiry_not_xml(self): # Code path coverage to test a policy that is not even valid XML - test_store = self.from_url(self.url, expiry_days=1) + test_store = self.from_url(self.url, expiry_days=1, validate_xml_policies=True) assert_raises(ValueError, test_store.create_array, 'test-expiry') @timed(0.1 + 0.05) diff --git a/test-requirements.txt b/test-requirements.txt index d2b201f3..c10b6262 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,6 @@ coverage funcsigs +lxml mock nose pbr From b0216df43252e22799b253e2b00052b5188dc798 Mon Sep 17 00:00:00 2001 From: Simon Ratcliffe Date: Wed, 24 Apr 2019 17:23:31 +0200 Subject: [PATCH 4/7] Pin lxml version --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c10b6262..5df7ec00 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,6 @@ coverage funcsigs -lxml +lxml==4.3.3 mock nose pbr From 1482ccf7b4258adcfc330dc59b9304020a15c437 Mon Sep 17 00:00:00 2001 From: Simon Ratcliffe Date: Thu, 25 Apr 2019 11:48:25 +0200 Subject: [PATCH 5/7] Generalise validator a little --- katdal/chunkstore_s3.py | 2 +- katdal/schemas/__init__.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/katdal/chunkstore_s3.py b/katdal/chunkstore_s3.py index 6faa7a28..bae12773 100644 --- a/katdal/chunkstore_s3.py +++ b/katdal/chunkstore_s3.py @@ -486,7 +486,7 @@ def create_array(self, array_name): if self.validate_xml_policies: if not schemas.has_lxml: raise ImportError("XML schema validation requires lxml to be installed.") - schemas.MINIMAL_LIFECYCLE_POLICY.validate(xml_payload) + schemas.validate('MINIMAL_LIFECYCLE_POLICY', xml_payload) b64_md5 = base64.b64encode(hashlib.md5(xml_payload.encode('utf-8')).digest()).decode('utf-8') lifecycle_headers = {'Content-Type': 'text/xml', 'Content-MD5': b64_md5} with self.request(None, 'PUT', url, params='lifecycle', data=xml_payload, headers=lifecycle_headers): diff --git a/katdal/schemas/__init__.py b/katdal/schemas/__init__.py index c3cb9fcd..200ba2a0 100644 --- a/katdal/schemas/__init__.py +++ b/katdal/schemas/__init__.py @@ -4,6 +4,7 @@ has_lxml = False +validators = {} try: from lxml import etree @@ -42,8 +43,16 @@ def validate(self, xml_string): return True +def validate(validator_name, string_to_validate): + try: + return validators[validator_name].validate(string_to_validate) + except KeyError: + raise KeyError("Specified validator {} doesn't map to an installed" + " schema.".format(validator_name)) + + for name in pkg_resources.resource_listdir(__name__, '.'): if name.endswith('.xsd') and has_lxml: xmlschema_doc = etree.parse(pkg_resources.resource_stream(__name__, name)) xml_validator = etree.XMLSchema(xmlschema_doc) - globals()[name[:-4].upper()] = ValidatorWithLog(xml_validator) + validators[name[:-4].upper()] = ValidatorWithLog(xml_validator) From 7e0ffd6c87c64b28a6616c142eedfcfc5f7b5014 Mon Sep 17 00:00:00 2001 From: Simon Ratcliffe Date: Thu, 25 Apr 2019 16:05:58 +0200 Subject: [PATCH 6/7] Include package data for schemas --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 0cdf1dd5..2de98238 100755 --- a/setup.py +++ b/setup.py @@ -35,6 +35,8 @@ author='Ludwig Schwardt', author_email='ludwig@ska.ac.za', packages=find_packages(), + package_data={'katdal': ['schemas/*']}, + include_package_data=True, scripts=[ 'scripts/h5list.py', 'scripts/h5toms.py', From ac78f20607311e120de0427cb920e10487e50369 Mon Sep 17 00:00:00 2001 From: Simon Ratcliffe Date: Fri, 17 May 2019 12:18:31 +0200 Subject: [PATCH 7/7] PR fixes - mostly around exception naming and handling --- katdal/chunkstore.py | 2 +- katdal/chunkstore_s3.py | 4 ++-- katdal/schemas/__init__.py | 8 ++++---- katdal/test/test_chunkstore_s3.py | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/katdal/chunkstore.py b/katdal/chunkstore.py index 72080b80..ad6938b8 100644 --- a/katdal/chunkstore.py +++ b/katdal/chunkstore.py @@ -38,7 +38,7 @@ class StoreUnavailable(OSError, ChunkStoreError): """Could not access underlying storage medium (offline, auth failed, etc).""" -class NotSupported(ChunkStoreError): +class UnsupportedStoreFeature(ChunkStoreError): """The underlying store does not support the requested operation (e.g. minio doesn't support lifecycle policies on buckets)""" diff --git a/katdal/chunkstore_s3.py b/katdal/chunkstore_s3.py index bae12773..ffd786d9 100644 --- a/katdal/chunkstore_s3.py +++ b/katdal/chunkstore_s3.py @@ -53,7 +53,7 @@ botocore = None from .chunkstore import (ChunkStore, StoreUnavailable, ChunkNotFound, BadChunk, - NotSupported, npy_header_and_body) + UnsupportedStoreFeature, npy_header_and_body) from .sensordata import to_str from . import schemas @@ -183,7 +183,7 @@ def _raise_for_status(response): if response.status_code == 404: raise ChunkNotFound(str(error)) elif response.status_code == 501: - raise_from(NotSupported(str(error)), error) + raise_from(UnsupportedStoreFeature(str(error)), error) else: raise StoreUnavailable(str(error)) diff --git a/katdal/schemas/__init__.py b/katdal/schemas/__init__.py index 200ba2a0..c4f65c28 100644 --- a/katdal/schemas/__init__.py +++ b/katdal/schemas/__init__.py @@ -1,7 +1,7 @@ """Makes packaged XSD schemas available as validators.""" import pkg_resources - +from future.utils import raise_from has_lxml = False validators = {} @@ -36,7 +36,7 @@ def validate(self, xml_string): try: xml_doc = etree.fromstring(bytes(bytearray(xml_string, encoding='utf-8'))) except etree.XMLSyntaxError as e: - raise ValueError(e) + raise_from(ValueError("Supplied string cannot be parsed as XML"), e) if not self.validator.validate(xml_doc): log = self.validator.error_log raise etree.DocumentInvalid(log.last_error) @@ -47,8 +47,8 @@ def validate(validator_name, string_to_validate): try: return validators[validator_name].validate(string_to_validate) except KeyError: - raise KeyError("Specified validator {} doesn't map to an installed" - " schema.".format(validator_name)) + raise_from(KeyError("Specified validator {} doesn't map to an installed" + " schema.".format(validator_name)), None) for name in pkg_resources.resource_listdir(__name__, '.'): diff --git a/katdal/test/test_chunkstore_s3.py b/katdal/test/test_chunkstore_s3.py index 8fa72f7d..49de308b 100644 --- a/katdal/test/test_chunkstore_s3.py +++ b/katdal/test/test_chunkstore_s3.py @@ -54,7 +54,7 @@ import requests from katdal.chunkstore_s3 import S3ChunkStore, _AWSAuth, read_array -from katdal.chunkstore import StoreUnavailable, NotSupported +from katdal.chunkstore import StoreUnavailable, UnsupportedStoreFeature from katdal.test.test_chunkstore import ChunkStoreTestBase # No expiration rule included @@ -242,11 +242,11 @@ def test_bucket_expiry(self): # NOTE: Minimum bucket expiry time is 1 day so real world testing is impractical. # We expect not supported since minio doesn't allow lifecycle policies test_store = self.from_url(self.url, expiry_days=1) - assert_raises(NotSupported, test_store.create_array, 'test-expiry') + assert_raises(UnsupportedStoreFeature, test_store.create_array, 'test-expiry') def test_bucket_expiry_with_validation(self): test_store = self.from_url(self.url, expiry_days=1, validate_xml_policies=True) - assert_raises(NotSupported, test_store.create_array, 'test-expiry') + assert_raises(UnsupportedStoreFeature, test_store.create_array, 'test-expiry') @mock.patch('katdal.chunkstore_s3._BASE_LIFECYCLE_POLICY', _INVALID_LIFECYCLE_POLICY) def test_bucket_expiry_invalid_schema(self):