diff --git a/README.markdown b/README.markdown index 32fbcc8..773a887 100644 --- a/README.markdown +++ b/README.markdown @@ -46,6 +46,20 @@ Dominik Moritz . For the current list of contributors see [github.com/ckan/datapusher/contributors](https://github.com/ckan/datapusher/contributors) +## Configuration Options + +- `HOST`: +- `PORT`: +- `MAX_CONTENT_LENGTH`: (Default: 10485760) +- `CHUNK_SIZE`: (Default: 16384) +- `CHUNK_INSERT_ROWS`: (Default: 250) +- `DOWNLOAD_TIMEOUT`: (Default: 30) +- `EXTERNAL_STORAGE`: Add an option to exclude CKAN API which mitigates [this issue](https://github.com/ckan/datapusher/issues/116) (Default: False) +- `SSL_VERIFY`: (Default: True) +- `TYPE_MAPPING`: +- `TYPES`: +- `GET_ROW_SET`: + ## Development To install DataPusher for development: diff --git a/datapusher/jobs.py b/datapusher/jobs.py index 3f6c309..5620059 100644 --- a/datapusher/jobs.py +++ b/datapusher/jobs.py @@ -33,6 +33,7 @@ CHUNK_SIZE = web.app.config.get('CHUNK_SIZE') or 16384 CHUNK_INSERT_ROWS = web.app.config.get('CHUNK_INSERT_ROWS') or 250 DOWNLOAD_TIMEOUT = web.app.config.get('DOWNLOAD_TIMEOUT') or 30 +EXTERNAL_STORAGE = web.app.config.get('EXTERNAL_STORAGE') or False if web.app.config.get('SSL_VERIFY') in ['False', 'FALSE', '0', False, 0]: SSL_VERIFY = False @@ -359,7 +360,8 @@ def push_to_datastore(task_id, input, dry_run=False): if resource.get('url_type') == 'upload': # If this is an uploaded file to CKAN, authenticate the request, # otherwise we won't get file from private resources - headers['Authorization'] = api_key + if not EXTERNAL_STORAGE: + headers['Authorization'] = api_key try: response = requests.get( url, diff --git a/deployment/datapusher_settings.py b/deployment/datapusher_settings.py index 4bac651..9fe8171 100644 --- a/deployment/datapusher_settings.py +++ b/deployment/datapusher_settings.py @@ -24,3 +24,7 @@ #LOG_FILE = '/tmp/ckan_service.log' STDERR = True + +# other + +#EXTERNAL_STORAGE = False \ No newline at end of file