CS-51 - Remove dependency on dynamodb.yml from cc_dynamodb3

.gitignore

@@ -23,7 +23,7 @@ sdist/
 var/
 *.egg-info/
 .installed.cfg
-*.egg
+*.egg*
 
 # Installer logs
 pip-log.txt

README.md

@@ -6,10 +6,9 @@ Here's a bullet-point summary:
 
 * provides a convenient model interface to query, scan, create, update and delete items
 * boto3 integration with conversion between dynamodb and pythonic data
-* parses table configuration as defined in a YAML file (see [tests/dynamodb.yml](tests/dynamodb.yml))
+* parses table configuration as defined in a terraform file (see [tests/dynamo_tables.tf](tests/dynamo_tables.tf))
 * namespaces tables so you can share the same configuration between different environments
 * gives you `Table` objects that have the schema and indexes loaded locally so you avoid extra lookups
-* direct calls to create or update tables by name as the configuration changes
 * optional ability to define non-indexed columns and types of data you expect to store
 
 **TODO: add back to Solano. Old cc_dynamodb was running.**
@@ -33,10 +32,11 @@ class TestModel(DynamoDBModel):
 
 
 cc_dynamodb3.set_config(
-    config_file_path='path/to/yaml/file.yml',
+    'path/to/file.tf',
     aws_access_key_id='<KEY>',
     aws_secret_access_key='<SECRET>',
-    namespace='dev_')
+    namespace='dev_',
+)
 
 obj = TestModel.create(agency_subdomain='test')  # calls PutItem
 obj.is_enabled = True
@@ -48,26 +48,19 @@ for obj in TestModel.all():
 ```
 And configuration:
 
-```yaml
-schemas:
-    test:  # note: no namespacing here
-        -
-            type: HashKey
-            name: agency_subdomain
-            data_type: STRING
-
-```
+https://www.terraform.io/docs/providers/aws/r/dynamodb_table.html
 
 Plain:
 
 ```python
 import cc_dynamodb3
 
 cc_dynamodb3.set_config(
-    config_file_path='path/to/yaml/file.yml',
+    'path/to/file.tf',
     aws_access_key_id='<KEY>',
     aws_secret_access_key='<SECRET>',
-    namespace='dev_')
+    namespace='dev_',
+)
 
 table = cc_dynamodb3.table.get_table('employment_screening_reports')
     # Returns the boto Table object
@@ -88,21 +81,21 @@ Returns the cached config. Calls `set_config` first if no cached config was foun
 
 ### `set_config(**kwargs)`
 
-Loads up the YAML configuration file and validates dynamodb connection details. The following are required, either set through the environment, or passed in as kwargs (to overwrite):
+Loads up the terraform configuration file and validates dynamodb connection details. The following are required, either set through the environment, or passed in as kwargs (to overwrite):
 
 * `namespace`, determines the table name prefix. Each repository using this library should have a unique namespace.
 * `aws_access_key_id` and `aws_secret_access_key`, the AWS connection credentials for boto's connection. Examples shown in [the tutorial](https://boto3.readthedocs.org/en/latest/guide/quickstart.html#configuration)
-* `table_config`, a path to the YAML file for table configuration.
+* `table_config`, a path to the terraform file for table configuration.
 
-### dynamodb.yml
+### dynamo_tables.tf
 
 This file contains the table schema for each table (required), and optional secondary indexes (`global_indexes`  or indexes (local secondary indexes).
 
 ### `set_redis_config(host='localhost', port=6379, db=3)`
 
 The headline is an example call. Redis caching is optional, but may greatly speed up your server performance.
 
-Redis caching is used to avoid parsing the YAML file every time `set_config()` is called.
+Redis caching is used to avoid parsing the config file every time `set_config()` is called.
 
 ## Usage
 
@@ -119,20 +112,13 @@ The following are all at the `cc_dynamodb3` top level. With the exception of `ge
     |------------------------------------------------------------------------------------------|
     | get_connection           | Returns a DynamoDBConnection even if credentials are invalid. |
     |------------------------------------------------------------------------------------------|
-    | get_table_columns        | Return known columns for a table and their data type.         |
-    |------------------------------------------------------------------------------------------|
     | query_table              | Provides a nicer interface to query a table than boto3        |
     |                          | default.                                                      |
     |------------------------------------------------------------------------------------------|
     | get_table                | Returns a dict with table and preloaded schema, plus columns. |
     |------------------------------------------------------------------------------------------|
     | list_table_names         | List known table names from configuration, without namespace. |
     |------------------------------------------------------------------------------------------|
-    | create_table             | Create table. Throws an error if table already exists.        |
-    |------------------------------------------------------------------------------------------|
-    | update_table             | Handles updating primary index and global secondary indexes.  |
-    |                          | Updates throughput and creates/deletes indexes.               |
-    |------------------------------------------------------------------------------------------|
 
 ## Mocks: `cc_dynamodb3.mocks`
 
@@ -157,20 +143,20 @@ Example:
 
 In your configuration file, e.g. `config.py`:
 
-    DYNAMODB_CONFIG_PATH = os.path.join(os.path.dirname(__file__), 'dynamodb.yml')
+    DYNAMODB_TABLE_TF = os.path.join(os.path.dirname(__file__), 'dynamodb_table.tf')
     DATABASE = dict(
+        DYNAMODB_TABLE_TF,
         namespace='test_',
-        table_config=DYNAMODB_CONFIG_PATH,
         aws_access_key_id='test',
         aws_secret_access_key='secret',
     )
 
 If you want to use [DynamoDB Local](http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Tools.DynamoDBLocal.html), just pass `host` as a parameter in the connection, e.g.:
 
     DATABASE = dict(
+        DYNAMODB_TABLE_TF,
         namespace='test_',
         host='localhost',
-        table_config=DYNAMODB_CONFIG_PATH,
         aws_access_key_id='test',
         aws_secret_access_key='secret',
     )
@@ -186,12 +172,16 @@ In your database file, e.g. `db.py`:
 Then you can use the library directly:
 
     import cc_dynamodb3
-    TABLE_NAME = 'some_table'
 
-    table = cc_dynamodb3.get_table(TABLE_NAME)
+    table = cc_dynamodb3.get_table('some_table')
     item = table.get_item(Key={'some_key': 'value'})
 
 ## Dynamodb Tutorial
 
 For more on boto3's `dynamodb` interface, please see [their guide](https://boto3.readthedocs.org/en/latest/guide/dynamodb.html).
 
+## Run the tests
+
+```
+$ py.test tests
+```

cc_dynamodb3/config.py

@@ -1,25 +1,31 @@
-import copy
-import json
 import os
+import redis
 
 from bunch import Bunch
-import redis
-import yaml
+from hcl_translator import dynamodb3_translator
 
 from .exceptions import ConfigurationError
 
 
-CONFIG_CACHE_KEY = 'cc_dynamodb3_yaml_config_cache'
+CONFIG_CACHE_KEY = 'cc_dynamodb3_config_cache'
 
-_config_file_path = None
-# Cache to avoid parsing YAML file repeatedly.
 _cached_config = None
 
 # Redis cache, optional but recommended
 # Example: dict(host='localhost', port=6379, db=3)
 _redis_config = dict()
 
 
+_dynamodb_tables = None
+
+
+def _get_tables(config):
+    global _dynamodb_tables
+    if _dynamodb_tables is None:
+        _dynamodb_tables = dynamodb3_translator(config)
+    return _dynamodb_tables
+
+
 def set_redis_config(redis_config):
     global _redis_config
     redis_config.setdefault('cache_seconds', 60)
@@ -46,30 +52,12 @@ def get_redis_cache():
 _redis_cache = get_redis_cache()
 
 
-def load_yaml_config():
-    global _config_file_path
-
-    redis_cache = get_redis_cache()
-    if redis_cache:
-        yaml_config = redis_cache.get(CONFIG_CACHE_KEY)
-        if yaml_config:
-            return json.loads(yaml_config)
-
-    with open(_config_file_path) as config_file:
-        yaml_config = yaml.load(config_file)
-        if redis_cache:
-            redis_config = get_redis_config()
-            redis_cache.setex(CONFIG_CACHE_KEY, redis_config['cache_seconds'], json.dumps(yaml_config))
-
-    return yaml_config
-
-
-def set_config(config_file_path, namespace=None, aws_access_key_id=False, aws_secret_access_key=False,
+def set_config(dynamodb_tf, namespace=None, aws_access_key_id=False, aws_secret_access_key=False,
                host=None, port=None, is_secure=None, log_extra_callback=None):
     """
     Set configuration. This is needed only once, globally, per-thread.
 
-    :param config_file_path: This is the path to the configuration file.
+    :param This is the path to the terraform configuration file.
     :param namespace: The global table namespace to be used for all tables
     :param aws_access_key_id: (optional) AWS key. boto can grab it from the instance metadata
     :param aws_secret_access_key: (optional) AWS secret. boto can grab it from the instance metadata
@@ -81,18 +69,14 @@ def set_config(config_file_path, namespace=None, aws_access_key_id=False, aws_se
     from .log import logger  # avoid circular import
 
     global _cached_config
-    global _config_file_path
-    _config_file_path = config_file_path
-
-    yaml_config = load_yaml_config()
+    global _dynamodb_table_info
 
     _cached_config = Bunch({
-        'yaml': yaml_config,
-        'namespace': namespace
-                        or os.environ.get('CC_DYNAMODB_NAMESPACE'),
-        'aws_access_key_id': aws_access_key_id if aws_access_key_id != False
+        'table_config': _get_tables(dynamodb_tf),
+        'namespace': namespace or os.environ.get('CC_DYNAMODB_NAMESPACE'),
+        'aws_access_key_id': aws_access_key_id if aws_access_key_id is not False
                                 else os.environ.get('CC_DYNAMODB_ACCESS_KEY_ID', False),
-        'aws_secret_access_key': aws_secret_access_key if aws_secret_access_key != False
+        'aws_secret_access_key': aws_secret_access_key if aws_secret_access_key is not False
                                     else os.environ.get('CC_DYNAMODB_SECRET_ACCESS_KEY', False),
         'host': host or os.environ.get('CC_DYNAMODB_HOST'),
         'port': port or os.environ.get('CC_DYNAMODB_PORT'),
@@ -138,16 +122,8 @@ def _validate_config():
             raise ConfigurationError(msg)
 
 
-def get_config(**kwargs):
+def get_config():
     global _cached_config
-
-    if not _cached_config:
-        # TODO: get_config() should never set_config()
-        # Since it's checking _cached_config, and won't set_config() if _cached_config is set,
-        # it really doesn't make sense that this ever get called if the config is already set.
-        # And get_config() with zero arguments when config is not set will cause TypeError.
-        # Makes far more sense for this to just always *only* get, and require set_config()
-        # be invoked before calling get_config().
-        set_config(**kwargs)
-
-    return Bunch(copy.deepcopy(_cached_config.toDict()))
+    if _cached_config is None:
+        raise ConfigurationError('set_config has to be called before get_config')
+    return _cached_config

cc_dynamodb3/mocks.py

@@ -1,9 +1,4 @@
-import cc_dynamodb3.table
-
-
-__all__ = [
-    'mock_table_with_data',
-]
+import cc_dynamodb3
 
 
 def mock_table_with_data(table_name, data):
@@ -16,9 +11,9 @@ def mock_table_with_data(table_name, data):
 
     len(table.scan())  # Expect 2 results
     """
-    table = cc_dynamodb3.table.create_table(table_name)
-
+    dynamodb = cc_dynamodb3.connection.get_connection()
+    dynamodb.create_table(**cc_dynamodb3.table.get_table_config(table_name))
+    table = dynamodb.Table(cc_dynamodb3.table.get_table_name(table_name))
     for item_data in data:
         table.put_item(Item=item_data)
-
     return table

cc_dynamodb3/models.py

@@ -87,7 +87,7 @@ def get(cls, consistent_read=False, **kwargs):
         :param kwargs: primary key fields.
         :return: instance of this model
         """
-        table_keys = [key['name'] for key in cls.get_schema()]
+        table_keys = [key['AttributeName'] for key in cls.get_schema()]
 
         if set(kwargs.keys()) != set(table_keys):
             raise exceptions.ValidationError('Invalid get kwargs: %s, expecting: %s' %
@@ -256,13 +256,12 @@ def to_json(self, role=None, context=None):
 
     @classmethod
     def get_schema(cls):
-        config_yaml = get_config().yaml
-        return config_yaml['schemas'][cls.TABLE_NAME]
+        return get_config().table_config.get_table(cls.TABLE_NAME)['KeySchema']
 
     def get_primary_key(self):
         """Return a dictionary used for cls.get by an item's primary key."""
         return dict(
-            (key['name'], self.item[key['name']])
+            (key['AttributeName'], self.item[key['AttributeName']])
             for key in self.get_schema()
         )
 
@@ -358,7 +357,7 @@ def update(self, skip_primary_key_check=False):
 
         if not skip_primary_key_check and self.has_changed_primary_key():
             raise exceptions.PrimaryKeyUpdateException(
-                    'Cannot change primary key, use %s.save(overwrite=True)' % self.TABLE_NAME)
+                'Cannot change primary key, use %s.save(overwrite=True)' % self.TABLE_NAME)
 
         response = self.table().update_item(
             Key=self.get_primary_key(),
@@ -422,8 +421,7 @@ def save(self, overwrite=False):
             log_data('save overwrite=True table=%s' % self.table().name,
                      extra=dict(
                          db_item=dict(self.item.items()),
-                         put_item_result=result,
-                     ),
+                         put_item_result=result),
                      logging_level='warning')
 
         if not overwrite: