diff --git a/datapusher/jobs.py b/datapusher/jobs.py index 3d06307..5c2dfb3 100644 --- a/datapusher/jobs.py +++ b/datapusher/jobs.py @@ -61,6 +61,7 @@ 'resource_update': '{ckan_url}/api/action/resource_update' } +BLACKLIST = web.app.config.get('BLACKLIST_URLS', '').split(' ') class HTTPError(util.JobError): """Exception that's raised if a job fails due to an HTTP problem.""" @@ -351,6 +352,12 @@ def push_to_datastore(task_id, input, dry_run=False): raise util.JobError( 'Only http, https, and ftp resources may be fetched.' ) + + for black_url in BLACKLIST: + if black_url and black_url in url: + raise util.JobError( + 'Resouce can not be fetched - Given URL is blacklisted: %s' % url + ) # fetch the resource data logger.info('Fetching from: {0}'.format(url))