From fcd798db7eb17c7230c09d714f53e073dd1d4768 Mon Sep 17 00:00:00 2001 From: Derrick Petzold Date: Fri, 29 Apr 2016 19:00:46 -0700 Subject: [PATCH] CS-51 - Remove dependency on dynamodb.yml from cc_dynamodb3 - Removed support for dynamodb.yml. The dynamodb terraform file is now the authorative source of truth for the dynamodb schema information. - Removed support for table creation and updates and there related tests as that is handled by terraform now. - Removed support for retrieving table column information. That will have to be done outside of cc_dynamodb3. --- .gitignore | 2 +- README.md | 52 +++--- cc_dynamodb3/config.py | 72 +++----- cc_dynamodb3/mocks.py | 13 +- cc_dynamodb3/models.py | 12 +- cc_dynamodb3/table.py | 275 +---------------------------- requirements.txt | 1 + setup.cfg | 3 + setup.py | 10 +- tests/conftest.py | 11 +- tests/dynamo_tables.tf | 89 ++++++++++ tests/factories/base.py | 11 +- tests/test_db_comparison.py | 10 +- tests/test_get_and_create_table.py | 13 +- tests/test_get_table_columns.py | 19 -- tests/test_hash_only_model.py | 2 +- tests/test_redis_config.py | 15 +- tests/test_update_table.py | 78 -------- 18 files changed, 195 insertions(+), 493 deletions(-) create mode 100644 setup.cfg create mode 100644 tests/dynamo_tables.tf delete mode 100644 tests/test_update_table.py diff --git a/.gitignore b/.gitignore index 418f003..6f32554 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,7 @@ sdist/ var/ *.egg-info/ .installed.cfg -*.egg +*.egg* # Installer logs pip-log.txt diff --git a/README.md b/README.md index 9e97c3b..8905d9b 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,9 @@ Here's a bullet-point summary: * provides a convenient model interface to query, scan, create, update and delete items * boto3 integration with conversion between dynamodb and pythonic data -* parses table configuration as defined in a YAML file (see [tests/dynamodb.yml](tests/dynamodb.yml)) +* parses table configuration as defined in a terraform file (see [tests/dynamo_tables.tf](tests/dynamo_tables.tf)) * namespaces tables so you can share the same configuration between different environments * gives you `Table` objects that have the schema and indexes loaded locally so you avoid extra lookups -* direct calls to create or update tables by name as the configuration changes * optional ability to define non-indexed columns and types of data you expect to store **TODO: add back to Solano. Old cc_dynamodb was running.** @@ -33,10 +32,11 @@ class TestModel(DynamoDBModel): cc_dynamodb3.set_config( - config_file_path='path/to/yaml/file.yml', + 'path/to/file.tf', aws_access_key_id='', aws_secret_access_key='', - namespace='dev_') + namespace='dev_', +) obj = TestModel.create(agency_subdomain='test') # calls PutItem obj.is_enabled = True @@ -48,15 +48,7 @@ for obj in TestModel.all(): ``` And configuration: -```yaml -schemas: - test: # note: no namespacing here - - - type: HashKey - name: agency_subdomain - data_type: STRING - -``` +https://www.terraform.io/docs/providers/aws/r/dynamodb_table.html Plain: @@ -64,10 +56,11 @@ Plain: import cc_dynamodb3 cc_dynamodb3.set_config( - config_file_path='path/to/yaml/file.yml', + 'path/to/file.tf', aws_access_key_id='', aws_secret_access_key='', - namespace='dev_') + namespace='dev_', +) table = cc_dynamodb3.table.get_table('employment_screening_reports') # Returns the boto Table object @@ -88,13 +81,13 @@ Returns the cached config. Calls `set_config` first if no cached config was foun ### `set_config(**kwargs)` -Loads up the YAML configuration file and validates dynamodb connection details. The following are required, either set through the environment, or passed in as kwargs (to overwrite): +Loads up the terraform configuration file and validates dynamodb connection details. The following are required, either set through the environment, or passed in as kwargs (to overwrite): * `namespace`, determines the table name prefix. Each repository using this library should have a unique namespace. * `aws_access_key_id` and `aws_secret_access_key`, the AWS connection credentials for boto's connection. Examples shown in [the tutorial](https://boto3.readthedocs.org/en/latest/guide/quickstart.html#configuration) -* `table_config`, a path to the YAML file for table configuration. +* `table_config`, a path to the terraform file for table configuration. -### dynamodb.yml +### dynamo_tables.tf This file contains the table schema for each table (required), and optional secondary indexes (`global_indexes` or indexes (local secondary indexes). @@ -102,7 +95,7 @@ This file contains the table schema for each table (required), and optional seco The headline is an example call. Redis caching is optional, but may greatly speed up your server performance. -Redis caching is used to avoid parsing the YAML file every time `set_config()` is called. +Redis caching is used to avoid parsing the config file every time `set_config()` is called. ## Usage @@ -119,8 +112,6 @@ The following are all at the `cc_dynamodb3` top level. With the exception of `ge |------------------------------------------------------------------------------------------| | get_connection | Returns a DynamoDBConnection even if credentials are invalid. | |------------------------------------------------------------------------------------------| - | get_table_columns | Return known columns for a table and their data type. | - |------------------------------------------------------------------------------------------| | query_table | Provides a nicer interface to query a table than boto3 | | | default. | |------------------------------------------------------------------------------------------| @@ -128,11 +119,6 @@ The following are all at the `cc_dynamodb3` top level. With the exception of `ge |------------------------------------------------------------------------------------------| | list_table_names | List known table names from configuration, without namespace. | |------------------------------------------------------------------------------------------| - | create_table | Create table. Throws an error if table already exists. | - |------------------------------------------------------------------------------------------| - | update_table | Handles updating primary index and global secondary indexes. | - | | Updates throughput and creates/deletes indexes. | - |------------------------------------------------------------------------------------------| ## Mocks: `cc_dynamodb3.mocks` @@ -157,10 +143,10 @@ Example: In your configuration file, e.g. `config.py`: - DYNAMODB_CONFIG_PATH = os.path.join(os.path.dirname(__file__), 'dynamodb.yml') + DYNAMODB_TABLE_TF = os.path.join(os.path.dirname(__file__), 'dynamodb_table.tf') DATABASE = dict( + DYNAMODB_TABLE_TF, namespace='test_', - table_config=DYNAMODB_CONFIG_PATH, aws_access_key_id='test', aws_secret_access_key='secret', ) @@ -168,9 +154,9 @@ In your configuration file, e.g. `config.py`: If you want to use [DynamoDB Local](http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Tools.DynamoDBLocal.html), just pass `host` as a parameter in the connection, e.g.: DATABASE = dict( + DYNAMODB_TABLE_TF, namespace='test_', host='localhost', - table_config=DYNAMODB_CONFIG_PATH, aws_access_key_id='test', aws_secret_access_key='secret', ) @@ -186,12 +172,16 @@ In your database file, e.g. `db.py`: Then you can use the library directly: import cc_dynamodb3 - TABLE_NAME = 'some_table' - table = cc_dynamodb3.get_table(TABLE_NAME) + table = cc_dynamodb3.get_table('some_table') item = table.get_item(Key={'some_key': 'value'}) ## Dynamodb Tutorial For more on boto3's `dynamodb` interface, please see [their guide](https://boto3.readthedocs.org/en/latest/guide/dynamodb.html). +## Run the tests + +``` +$ py.test tests +``` diff --git a/cc_dynamodb3/config.py b/cc_dynamodb3/config.py index 2af02f6..4973468 100644 --- a/cc_dynamodb3/config.py +++ b/cc_dynamodb3/config.py @@ -1,18 +1,14 @@ -import copy -import json import os +import redis from bunch import Bunch -import redis -import yaml +from hcl_translator import dynamodb3_translator from .exceptions import ConfigurationError -CONFIG_CACHE_KEY = 'cc_dynamodb3_yaml_config_cache' +CONFIG_CACHE_KEY = 'cc_dynamodb3_config_cache' -_config_file_path = None -# Cache to avoid parsing YAML file repeatedly. _cached_config = None # Redis cache, optional but recommended @@ -20,6 +16,16 @@ _redis_config = dict() +_dynamodb_tables = None + + +def _get_tables(config): + global _dynamodb_tables + if _dynamodb_tables is None: + _dynamodb_tables = dynamodb3_translator(config) + return _dynamodb_tables + + def set_redis_config(redis_config): global _redis_config redis_config.setdefault('cache_seconds', 60) @@ -46,30 +52,12 @@ def get_redis_cache(): _redis_cache = get_redis_cache() -def load_yaml_config(): - global _config_file_path - - redis_cache = get_redis_cache() - if redis_cache: - yaml_config = redis_cache.get(CONFIG_CACHE_KEY) - if yaml_config: - return json.loads(yaml_config) - - with open(_config_file_path) as config_file: - yaml_config = yaml.load(config_file) - if redis_cache: - redis_config = get_redis_config() - redis_cache.setex(CONFIG_CACHE_KEY, redis_config['cache_seconds'], json.dumps(yaml_config)) - - return yaml_config - - -def set_config(config_file_path, namespace=None, aws_access_key_id=False, aws_secret_access_key=False, +def set_config(dynamodb_tf, namespace=None, aws_access_key_id=False, aws_secret_access_key=False, host=None, port=None, is_secure=None, log_extra_callback=None): """ Set configuration. This is needed only once, globally, per-thread. - :param config_file_path: This is the path to the configuration file. + :param This is the path to the terraform configuration file. :param namespace: The global table namespace to be used for all tables :param aws_access_key_id: (optional) AWS key. boto can grab it from the instance metadata :param aws_secret_access_key: (optional) AWS secret. boto can grab it from the instance metadata @@ -81,18 +69,14 @@ def set_config(config_file_path, namespace=None, aws_access_key_id=False, aws_se from .log import logger # avoid circular import global _cached_config - global _config_file_path - _config_file_path = config_file_path - - yaml_config = load_yaml_config() + global _dynamodb_table_info _cached_config = Bunch({ - 'yaml': yaml_config, - 'namespace': namespace - or os.environ.get('CC_DYNAMODB_NAMESPACE'), - 'aws_access_key_id': aws_access_key_id if aws_access_key_id != False + 'table_config': _get_tables(dynamodb_tf), + 'namespace': namespace or os.environ.get('CC_DYNAMODB_NAMESPACE'), + 'aws_access_key_id': aws_access_key_id if aws_access_key_id is not False else os.environ.get('CC_DYNAMODB_ACCESS_KEY_ID', False), - 'aws_secret_access_key': aws_secret_access_key if aws_secret_access_key != False + 'aws_secret_access_key': aws_secret_access_key if aws_secret_access_key is not False else os.environ.get('CC_DYNAMODB_SECRET_ACCESS_KEY', False), 'host': host or os.environ.get('CC_DYNAMODB_HOST'), 'port': port or os.environ.get('CC_DYNAMODB_PORT'), @@ -138,16 +122,8 @@ def _validate_config(): raise ConfigurationError(msg) -def get_config(**kwargs): +def get_config(): global _cached_config - - if not _cached_config: - # TODO: get_config() should never set_config() - # Since it's checking _cached_config, and won't set_config() if _cached_config is set, - # it really doesn't make sense that this ever get called if the config is already set. - # And get_config() with zero arguments when config is not set will cause TypeError. - # Makes far more sense for this to just always *only* get, and require set_config() - # be invoked before calling get_config(). - set_config(**kwargs) - - return Bunch(copy.deepcopy(_cached_config.toDict())) + if _cached_config is None: + raise ConfigurationError('set_config has to be called before get_config') + return _cached_config diff --git a/cc_dynamodb3/mocks.py b/cc_dynamodb3/mocks.py index 97165af..6efdadd 100644 --- a/cc_dynamodb3/mocks.py +++ b/cc_dynamodb3/mocks.py @@ -1,9 +1,4 @@ -import cc_dynamodb3.table - - -__all__ = [ - 'mock_table_with_data', -] +import cc_dynamodb3 def mock_table_with_data(table_name, data): @@ -16,9 +11,9 @@ def mock_table_with_data(table_name, data): len(table.scan()) # Expect 2 results """ - table = cc_dynamodb3.table.create_table(table_name) - + dynamodb = cc_dynamodb3.connection.get_connection() + dynamodb.create_table(**cc_dynamodb3.table.get_table_config(table_name)) + table = dynamodb.Table(cc_dynamodb3.table.get_table_name(table_name)) for item_data in data: table.put_item(Item=item_data) - return table diff --git a/cc_dynamodb3/models.py b/cc_dynamodb3/models.py index f9998b6..46df615 100644 --- a/cc_dynamodb3/models.py +++ b/cc_dynamodb3/models.py @@ -87,7 +87,7 @@ def get(cls, consistent_read=False, **kwargs): :param kwargs: primary key fields. :return: instance of this model """ - table_keys = [key['name'] for key in cls.get_schema()] + table_keys = [key['AttributeName'] for key in cls.get_schema()] if set(kwargs.keys()) != set(table_keys): raise exceptions.ValidationError('Invalid get kwargs: %s, expecting: %s' % @@ -256,13 +256,12 @@ def to_json(self, role=None, context=None): @classmethod def get_schema(cls): - config_yaml = get_config().yaml - return config_yaml['schemas'][cls.TABLE_NAME] + return get_config().table_config.get_table(cls.TABLE_NAME)['KeySchema'] def get_primary_key(self): """Return a dictionary used for cls.get by an item's primary key.""" return dict( - (key['name'], self.item[key['name']]) + (key['AttributeName'], self.item[key['AttributeName']]) for key in self.get_schema() ) @@ -358,7 +357,7 @@ def update(self, skip_primary_key_check=False): if not skip_primary_key_check and self.has_changed_primary_key(): raise exceptions.PrimaryKeyUpdateException( - 'Cannot change primary key, use %s.save(overwrite=True)' % self.TABLE_NAME) + 'Cannot change primary key, use %s.save(overwrite=True)' % self.TABLE_NAME) response = self.table().update_item( Key=self.get_primary_key(), @@ -422,8 +421,7 @@ def save(self, overwrite=False): log_data('save overwrite=True table=%s' % self.table().name, extra=dict( db_item=dict(self.item.items()), - put_item_result=result, - ), + put_item_result=result), logging_level='warning') if not overwrite: diff --git a/cc_dynamodb3/table.py b/cc_dynamodb3/table.py index 6822963..f545e1f 100644 --- a/cc_dynamodb3/table.py +++ b/cc_dynamodb3/table.py @@ -2,132 +2,10 @@ import operator from boto3.dynamodb.conditions import Key, Attr -from botocore.exceptions import ClientError from .config import get_config from .connection import get_connection -from .exceptions import ( - TableAlreadyExistsException, - UpdateTableException, - UnknownTableException, -) -from .log import log_data - - -def _build_key_type(key_type): - if key_type == 'HashKey': - return 'HASH' - if key_type == 'RangeKey': - return 'RANGE' - raise NotImplementedError('Unknown Key Type: %s' % key_type) - - -def _build_key_schema(keys_config): - return [ - { - 'KeyType': _build_key_type(key['type']), - 'AttributeName': key['name'], - } for key in keys_config - ] - - -def _build_attribute_definitions(keys_config): - return [ - { - 'AttributeName': key['name'], - 'AttributeType': key['data_type'][0], - } for key in keys_config - ] - - -def _build_index_type(index_type): - # Valid values: 'ALL'|'KEYS_ONLY'|'INCLUDE' - if index_type not in ('AllIndex', 'GlobalAllIndex'): - raise NotImplementedError('TODO: support KEYS_ONLY and INCLUDE with Projection') - return 'ALL' - - -def _get_table_metadata(table_name): - config = get_config().yaml - - try: - keys_config = config['schemas'][table_name] - except KeyError: - log_data('Unknown Table', - extra=dict(table_name=table_name, - config=config), - logging_level='exception') - raise UnknownTableException('Unknown table: %s' % table_name) - - metadata = dict( - KeySchema=_build_key_schema(keys_config), - AttributeDefinitions=_build_attribute_definitions(keys_config) - ) - - global_indexes_config = config.get('global_indexes', {}).get(table_name, []) - indexes_config = config.get('indexes', {}).get(table_name, []) - - if indexes_config: - lsis = [] - for lsi_config in indexes_config: - lsis.append({ - 'IndexName': lsi_config['name'], - 'KeySchema': [ - { - 'AttributeName': part['name'], - 'KeyType': _build_key_type(part['type']), - } - for part in lsi_config['parts'] - ], - 'Projection': { - 'ProjectionType': _build_index_type(lsi_config['type']), - }, - }) - attributes = [] - for index in indexes_config: - for attribute in index['parts']: - attributes.append(attribute) - metadata['AttributeDefinitions'] += _build_attribute_definitions(attributes) - metadata.update(LocalSecondaryIndexes=lsis) - - if global_indexes_config: - gsis = [] - for gsi_config in global_indexes_config: - formatted = _get_or_default_throughput(gsi_config.get('throughput') or False) - provisioned_throughput = formatted['ProvisionedThroughput'] - gsis.append({ - 'IndexName': gsi_config['name'], - 'KeySchema': [ - { - 'AttributeName': part['name'], - 'KeyType': _build_key_type(part['type']), - } - for part in gsi_config['parts'] - ], - 'Projection': { - 'ProjectionType': _build_index_type(gsi_config['type']), - }, - 'ProvisionedThroughput': provisioned_throughput, - }) - attributes = [] - for index in global_indexes_config: - for attribute in index['parts']: - attributes.append(attribute) - metadata['AttributeDefinitions'] += _build_attribute_definitions(attributes) - metadata.update(GlobalSecondaryIndexes=gsis) - - # Unique-fy AttributeDefinitions - attribute_definitions = dict() - for attribute in metadata['AttributeDefinitions']: - if (attribute['AttributeName'] in attribute_definitions and - attribute['AttributeType'] != attribute_definitions[attribute['AttributeName']]['AttributeType']): - raise ValueError('Mismatched attribute type for %s. Found: %s and %s' % - (attribute['AttributeName'], attribute['AttributeType'], - attribute_definitions[attribute['AttributeName']]['AttributeType'])) - attribute_definitions[attribute['AttributeName']] = attribute - metadata['AttributeDefinitions'] = attribute_definitions.values() - - return metadata +from .exceptions import UnknownTableException def get_table_name(table_name): @@ -163,21 +41,6 @@ def get_table_index(table_name, index_name): return index -def get_table_columns(table_name): - """Return known columns for a table and their data type.""" - config = get_config().yaml - try: - return dict( - (column_name, column_type) - for column_name, column_type in config['columns'][table_name].items()) - except KeyError: - log_data('Unknown Table', - extra=dict(table_name=table_name, - config=config), - logging_level='exception') - raise UnknownTableException('Unknown table: %s' % table_name) - - def get_table(table_name, connection=None): """Returns a dict with table and preloaded schema, plus columns. @@ -333,141 +196,13 @@ def query_all_in_table(table_name_or_class, *args, **kwargs): def list_table_names(): """List known table names from configuration, without namespace.""" - return get_config().yaml['schemas'].keys() - - -def _get_or_default_throughput(throughput): - if throughput is False: - config = get_config() - throughput = config.yaml['default_throughput'] - - if not throughput: - return dict() - return dict( - ProvisionedThroughput=dict( - ReadCapacityUnits=throughput['read'], - WriteCapacityUnits=throughput['write'], - ) - ) + return get_config().table_config.list_table_names() -def _get_table_init_data(table_name, throughput): +def get_table_config(table_name): + config = get_config() init_data = dict( TableName=get_table_name(table_name), ) - - init_data.update(_get_table_metadata(table_name)) - init_data.update(_get_or_default_throughput(throughput)) - + init_data.update(config.table_config.get_table(table_name)) return init_data - - -def create_table(table_name, connection=None, throughput=False): - """Create table. Throws an error if table already exists.""" - dynamodb = connection or get_connection() - init_data = _get_table_init_data(table_name, throughput=throughput) - try: - db_table = dynamodb.create_table(**init_data) - if isinstance(db_table, dict): - # We must be using moto's crappy mocking - db_table = dynamodb.Table(init_data['TableName']) - - db_table.meta.client.get_waiter('table_exists').wait(TableName=init_data['TableName']) - log_data('create_table: %s' % table_name, - extra=dict(status='created table', - table_name=table_name), - logging_level='info') - return db_table - except ClientError as e: - if (e.response['ResponseMetadata']['HTTPStatusCode'] == 400 and - e.response['Error']['Code'] == 'ResourceInUseException'): - log_data('Called create_table("%s"), but already exists: %s' % - (table_name, e.response), - extra=dict(table_name=table_name), - logging_level='warning') - raise TableAlreadyExistsException(response=e.response) - raise e - - -def _validate_schema(table_name, upstream_schema, local_schema): - """Raise error if primary index (schema) is not the same as upstream""" - if sorted(upstream_schema, key=lambda i: i['AttributeName']) != sorted(local_schema, key=lambda i: i['AttributeName']): - msg = 'Mismatched schema: %s VS %s' % (upstream_schema, local_schema) - log_data(msg, logging_level='warning') - raise UpdateTableException(msg) - - -def update_table(table_name, connection=None, throughput=False): - """ - Update existing table. - - Handles updating primary index and global secondary indexes. - Updates throughput and creates/deletes indexes. - - :param table_name: unprefixed table name - :param connection: optional dynamodb connection, to avoid creating one - :param throughput: a dict, e.g. {'read': 10, 'write': 10} - :return: the updated boto Table - """ - db_table = get_table(table_name, connection=connection) - try: - db_table.load() - except ClientError as e: - if (e.response['ResponseMetadata']['HTTPStatusCode'] == 400 and - e.response['Error']['Code'] == 'ResourceNotFoundException'): - raise UnknownTableException('Unknown table: %s' % table_name) - - local_metadata = _get_table_metadata(table_name) - _validate_schema(table_name, upstream_schema=db_table.key_schema, local_schema=local_metadata['KeySchema']) - - # Update existing primary index throughput - if throughput: - formatted = _get_or_default_throughput(throughput) - if formatted['ProvisionedThroughput'] != db_table.provisioned_throughput: - db_table.update(**formatted) - - local_global_indexes_by_name = dict((i['IndexName'], i) for i in local_metadata.get('GlobalSecondaryIndexes', [])) - upstream_global_indexes_by_name = dict((i['IndexName'], i) for i in (db_table.global_secondary_indexes or [])) - - gsi_updates = [] - - for index_name, index in local_global_indexes_by_name.items(): - if index_name not in upstream_global_indexes_by_name: - log_data('Creating GSI %s for %s' % (index_name, table_name), - logging_level='info') - gsi_updates.append({ - 'Create': index, - }) - else: - upstream_index = upstream_global_indexes_by_name[index_name] - if (index['ProvisionedThroughput'].get('ReadCapacityUnits') == - upstream_index['ProvisionedThroughput'].get('ReadCapacityUnits')) and \ - (index['ProvisionedThroughput'].get('WriteCapacityUnits') == - upstream_index['ProvisionedThroughput'].get('WriteCapacityUnits')): - continue - gsi_updates.append({ - 'Update': { - 'IndexName': index_name, - 'ProvisionedThroughput': index['ProvisionedThroughput'] - }, - }) - log_data('Updating GSI %s throughput for %s to %s' % (index_name, table_name, index['ProvisionedThroughput']), - logging_level='info') - - for index_name in upstream_global_indexes_by_name.keys(): - if index_name not in local_global_indexes_by_name: - log_data('Deleting GSI %s for %s' % (index_name, table_name), - logging_level='info') - gsi_updates.append({ - 'Delete': { - 'IndexName': index_name, - } - }) - - if gsi_updates: - db_table.update(AttributeDefinitions=local_metadata['AttributeDefinitions'], - GlobalSecondaryIndexUpdates=gsi_updates) - log_data('update_table: %s' % table_name, extra=dict(status='updated table', - table_name=table_name), - logging_level='info') - return db_table diff --git a/requirements.txt b/requirements.txt index 56bb16a..ac4e596 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ boto3==1.2.2 PyYAML>=3.10 schematics==1.1.1 redis==2.10.5 +-e git+git@github.com:clearcare/cc_hcl_translator.git@94f633fbb3412aeb7bba5dd18063845f4ed751a5#egg=hcl_translator diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..c25e181 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 160 +ignore = E131 diff --git a/setup.py b/setup.py index d2f086c..b99b2f4 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,19 @@ from setuptools import setup, find_packages setup( - name = 'cc_dynamodb3', + name='cc_dynamodb3', packages=find_packages(), install_requires=[ 'bunch>=1.0.1', 'boto3>=1.2.2', - 'PyYAML==3.10', 'schematics==1.1.1', ], + dependency_links=[ + 'git+https://github.com/clearcare/cc_hcl_translator.git@0.1.0#egg=hcl_translator', + ], tests_require=['pytest', 'mock', 'factory_boy', 'moto'], - version = '0.6.14', - description = 'A dynamodb common configuration abstraction', + version='1.0.4', + description='A dynamodb common configuration abstraction', author='Paul Craciunoiu', author_email='pcraciunoiu@clearcareonline.com', url='https://github.com/clearcare/cc_dynamodb3', diff --git a/tests/conftest.py b/tests/conftest.py index f612684..c72bcaa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,17 +5,18 @@ import pytest -AWS_DYNAMODB_CONFIG_PATH = os.path.join(os.path.dirname(__file__), 'dynamodb.yml') +AWS_DYNAMODB_TF = os.path.join(os.path.dirname(__file__), 'dynamo_tables.tf') @pytest.fixture(scope='function', autouse=True) def fake_config(): - import cc_dynamodb3.config + import cc_dynamodb3 cc_dynamodb3.config.set_config( - config_file_path=AWS_DYNAMODB_CONFIG_PATH, + AWS_DYNAMODB_TF, aws_access_key_id='', aws_secret_access_key='', - namespace='dev_') + namespace='dev_', + ) @pytest.fixture(scope='function', autouse=True) @@ -45,4 +46,4 @@ def mock_db(request): 'recommend_score': '3' }, ], -} \ No newline at end of file +} diff --git a/tests/dynamo_tables.tf b/tests/dynamo_tables.tf new file mode 100644 index 0000000..db2b7d8 --- /dev/null +++ b/tests/dynamo_tables.tf @@ -0,0 +1,89 @@ +resource "aws_dynamodb_table" "nps_survey" { + name = "nps_survey" + read_capacity = "5" + write_capacity = "5" + hash_key = "agency_id" + range_key = "profile_id" + attribute { + name = "agency_id" + type = "N" + } + attribute { + name = "profile_id" + type = "N" + } +} + + +resource "aws_dynamodb_table" "hash_only" { + name = "hash_only" + read_capacity = "5" + write_capacity = "5" + hash_key = "agency_subdomain" + global_secondary_index { + name = "HashOnlyExternalId" + hash_key = "external_id" + read_capacity = "15" + write_capacity = "15" + projection_type = "ALL" + } + attribute { + name = "agency_subdomain" + type = "S" + } + attribute { + name = "external_id" + type = "S" + } +} + + +resource "aws_dynamodb_table" "map_field" { + name = "map_field" + read_capacity = "5" + write_capacity = "5" + hash_key = "agency_subdomain" + attribute { + name = "agency_subdomain" + type = "S" + } +} + + +resource "aws_dynamodb_table" "change_in_condition" { + name = "change_in_condition" + read_capacity = "10" + write_capacity = "10" + hash_key = "carelog_id" + range_key = "time" + global_secondary_index { + name = "SavedInRDB" + hash_key = "saved_in_rdb" + range_key = "time" + read_capacity = "15" + write_capacity = "15" + projection_type = "ALL" + } + local_secondary_index { + name = "SessionId" + hash_key = "carelog_id" + range_key = "session_id" + projection_type = "ALL" + } + attribute { + name = "carelog_id" + type = "N" + } + attribute { + name = "time" + type = "N" + } + attribute { + name = "saved_in_rdb" + type = "N" + } + attribute { + name = "session_id" + type = "N" + } +} diff --git a/tests/factories/base.py b/tests/factories/base.py index 9305df7..259c891 100644 --- a/tests/factories/base.py +++ b/tests/factories/base.py @@ -1,15 +1,14 @@ -from cc_dynamodb3.table import create_table - import factory +import cc_dynamodb3 + class BaseFactory(factory.Factory): @classmethod def _create(cls, model_class, *args, **kwargs): - inst = model_class.create(**kwargs) - return inst + return model_class.create(**kwargs) @classmethod def create_table(cls): - return create_table(cls._meta.model.TABLE_NAME) - + dynamodb = cc_dynamodb3.connection.get_connection() + return dynamodb.create_table(**cc_dynamodb3.table.get_table_config(cls._meta.model.TABLE_NAME)) diff --git a/tests/test_db_comparison.py b/tests/test_db_comparison.py index 4a41548..b1904f5 100644 --- a/tests/test_db_comparison.py +++ b/tests/test_db_comparison.py @@ -1,8 +1,10 @@ +from __future__ import absolute_import + import datetime from cc_dynamodb3.models import return_different_fields_except -from factories.hash_only_model import HashOnlyModelFactory +from .factories.hash_only_model import HashOnlyModelFactory def test_return_different_fields_except_should_ignore_and_return_true(): @@ -23,6 +25,6 @@ def test_return_different_fields_except_should_return_diff(): assert obj1.created != obj2.created assert return_different_fields_except(obj1.item, obj2.item, ['updated']) == dict( - old=dict(created=obj2.item['created']), - new=dict(created=obj1.item['created']) - ) \ No newline at end of file + old=dict(created=obj2.item['created']), + new=dict(created=obj1.item['created']) + ) diff --git a/tests/test_get_and_create_table.py b/tests/test_get_and_create_table.py index bee7f0d..833e885 100644 --- a/tests/test_get_and_create_table.py +++ b/tests/test_get_and_create_table.py @@ -1,3 +1,8 @@ +import pytest + +import cc_dynamodb3.exceptions +import cc_dynamodb3.table + from conftest import DYNAMODB_FIXTURES from cc_dynamodb3.mocks import mock_table_with_data @@ -5,6 +10,7 @@ def test_mock_create_table_implements_table_scan(): data = DYNAMODB_FIXTURES['nps_survey'] data_by_profile_id = {i['profile_id']: i for i in data} + table = mock_table_with_data('nps_survey', data) results = list(table.scan()['Items']) @@ -14,4 +20,9 @@ def test_mock_create_table_implements_table_scan(): item = data_by_profile_id[result.get('profile_id')] assert item['agency_id'] == result.get('agency_id') assert item['recommend_score'] == result.get('recommend_score') - assert item.get('favorite') == result.get('favorite') \ No newline at end of file + assert item.get('favorite') == result.get('favorite') + + +def test_get_dynamodb_table_unknown_table_raises_exception(): + with pytest.raises(cc_dynamodb3.exceptions.UnknownTableException): + cc_dynamodb3.table.get_table('invalid_table') diff --git a/tests/test_get_table_columns.py b/tests/test_get_table_columns.py index 4d41cc5..e69de29 100644 --- a/tests/test_get_table_columns.py +++ b/tests/test_get_table_columns.py @@ -1,19 +0,0 @@ -import pytest - -import cc_dynamodb3.exceptions -import cc_dynamodb3.table - - -def test_get_dynamodb_table_unknown_table_raises_exception(): - with pytest.raises(cc_dynamodb3.exceptions.UnknownTableException): - cc_dynamodb3.table.get_table('invalid_table') - - -def test_get_dynamodb_columns_unknown_table_raises_exception(): - with pytest.raises(cc_dynamodb3.exceptions.UnknownTableException): - cc_dynamodb3.table.get_table_columns('invalid_table') - - -def test_get_dynamodb_table_columns_should_return_columns(): - columns = cc_dynamodb3.table.get_table_columns('nps_survey') - assert set(columns.keys()) == {'favorite', 'change', 'comments', 'recommend_score'} diff --git a/tests/test_hash_only_model.py b/tests/test_hash_only_model.py index 6bf8137..09b34e6 100644 --- a/tests/test_hash_only_model.py +++ b/tests/test_hash_only_model.py @@ -106,4 +106,4 @@ def test_negative_timestamp(): obj = HashOnlyModel.all().next() assert obj.created.year == long_ago.year - assert obj.item['created'] < 0 \ No newline at end of file + assert obj.item['created'] < 0 diff --git a/tests/test_redis_config.py b/tests/test_redis_config.py index d344d58..678ae52 100644 --- a/tests/test_redis_config.py +++ b/tests/test_redis_config.py @@ -2,27 +2,25 @@ import mock -from .conftest import AWS_DYNAMODB_CONFIG_PATH +from .conftest import AWS_DYNAMODB_TF -@mock.patch('cc_dynamodb3.config.yaml.load') @mock.patch('cc_dynamodb3.config.get_redis_cache') -def test_load_with_redis_does_not_call_yaml_load(get_redis_cache, yaml_load): +def test_load_with_redis_does_not_call_yaml_load(get_redis_cache): redis_mock = mock.Mock() redis_mock.get = lambda key: '{"foo": "bar"}' get_redis_cache.return_value = redis_mock cc_dynamodb3.config.set_config( - config_file_path='/path/to/file.yaml', + '/path/to/file.tf', aws_access_key_id='', aws_secret_access_key='', namespace='dev_') config = cc_dynamodb3.config.get_config() - assert not yaml_load.called + assert redis_mock.setex.called == 0 assert config.aws_access_key_id == '' - assert config.yaml['foo'] == 'bar' @mock.patch.object(cc_dynamodb3.config, '_redis_config') @@ -35,13 +33,12 @@ def test_load_with_redis_calls_yaml_load_if_cache_miss(get_redis_cache, _redis_c cc_dynamodb3.config.set_redis_config(dict()) cc_dynamodb3.config.set_config( - config_file_path=AWS_DYNAMODB_CONFIG_PATH, + AWS_DYNAMODB_TF, aws_access_key_id='', aws_secret_access_key='', namespace='dev_') config = cc_dynamodb3.config.get_config() - assert redis_mock.setex.called + assert redis_mock.setex.called == 0 assert config.aws_access_key_id == '' - assert config.yaml['default_throughput'] == {'read': 10, 'write': 10} diff --git a/tests/test_update_table.py b/tests/test_update_table.py deleted file mode 100644 index 9c71eb2..0000000 --- a/tests/test_update_table.py +++ /dev/null @@ -1,78 +0,0 @@ -import cc_dynamodb3.config -import cc_dynamodb3.exceptions -import cc_dynamodb3.table - -import mock -import pytest - - -def test_update_table_should_raise_if_table_doesnt_exist(): - with pytest.raises(cc_dynamodb3.exceptions.UnknownTableException): - cc_dynamodb3.table.update_table('change_in_condition') - - -def test_update_table_should_not_update_if_same_throughput(): - cc_dynamodb3.table.create_table('change_in_condition') - cc_dynamodb3.table.update_table('change_in_condition') - - table = cc_dynamodb3.table.get_table('change_in_condition') - table.load() - # Ensure the throughput has been updated - assert table.provisioned_throughput == {'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10, - 'NumberOfDecreasesToday': 0} - assert len(table.global_secondary_indexes) == 1 - assert table.global_secondary_indexes[0]['ProvisionedThroughput'] == {'WriteCapacityUnits': 15, - 'ReadCapacityUnits': 15} - - -def test_update_table_should_create_delete_gsi(): - cc_dynamodb3.table.create_table('change_in_condition') - - original_config = cc_dynamodb3.config.get_config() - patcher = mock.patch('cc_dynamodb3.table.get_config') - mock_config = patcher.start() - original_config.yaml['global_indexes']['change_in_condition'] = [{ - 'parts': [ - {'type': 'HashKey', 'name': 'rdb_id', 'data_type': 'NUMBER'}, - {'type': 'RangeKey', 'name': 'session_id', 'data_type': 'NUMBER'}], - 'type': 'GlobalAllIndex', - 'name': 'RdbID', - }] - mock_config.return_value = original_config - - cc_dynamodb3.table.update_table('change_in_condition', throughput={'read': 55, 'write': 44}) - mock_config.stop() - - table = cc_dynamodb3.table.get_table('change_in_condition') - table.load() - # Ensure the throughput has been updated - assert table.provisioned_throughput == {'ReadCapacityUnits': 55, 'WriteCapacityUnits': 44} - assert len(table.global_secondary_indexes) == 1 - assert table.global_secondary_indexes[0]['IndexName'] == 'RdbID' - assert table.global_secondary_indexes[0]['ProvisionedThroughput'] == {'WriteCapacityUnits': 10, - 'ReadCapacityUnits': 10} - - -def test_update_table_should_update_gsi(): - cc_dynamodb3.table.create_table('change_in_condition') - - original_config = cc_dynamodb3.config.get_config() - patcher = mock.patch('cc_dynamodb3.table.get_config') - mock_config = patcher.start() - original_config.yaml['global_indexes']['change_in_condition'][0]['throughput'] = { - 'read': 20, - 'write': 20, - } - mock_config.return_value = original_config - - cc_dynamodb3.table.update_table('change_in_condition') - mock_config.stop() - - table = cc_dynamodb3.table.get_table('change_in_condition') - table.load() - # Ensure the primary throughput has not been been updated - assert table.provisioned_throughput == {'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10, - 'NumberOfDecreasesToday': 0} - # ... but the GSI has been - assert table.global_secondary_indexes[0]['ProvisionedThroughput'] == {'WriteCapacityUnits': 20, - 'ReadCapacityUnits': 20}