From 4f680d9468f6e738db58f1ca0dad5eb5a4499907 Mon Sep 17 00:00:00 2001 From: Scott Reynen Date: Wed, 11 Mar 2020 07:10:13 -0600 Subject: [PATCH 1/3] initial API --- .gitignore | 4 ++ README.md | 20 ++++++++- dev_requirements.txt | 1 + export_rsvps.py | 84 +++++++++++++++++++++++++++++++++++++ requirements.txt | 3 ++ settings.py.template | 11 +++++ validate_key.py | 56 +++++++++++++++++++++++++ zappa_settings.yml.template | 37 ++++++++++++++++ 8 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 dev_requirements.txt create mode 100644 export_rsvps.py create mode 100644 requirements.txt create mode 100644 settings.py.template create mode 100644 validate_key.py create mode 100644 zappa_settings.yml.template diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ba76317 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.pyc +settings.py +venv +zappa_settings.yml diff --git a/README.md b/README.md index de70256..137014d 100644 --- a/README.md +++ b/README.md @@ -1 +1,19 @@ -# ak-partner-rsvp \ No newline at end of file +# ActionKit Partner RSVP Export + +This is a tool to allow partners to export their own sourced RSVPs from ActionKit event campaigns. It has two parts: a static site and an API. + +## Static site + +The static site in the repo is MoveOn-specific. If you're using this with a different ActionKit instance, you'd of course want to change the HTML and CSS to match your organization's brand, and also edit the apiRoot value at the top of static/index.js. + +### Deploy + +To deploy the static site, simply put the files on any web host. At MoveOn, we use S3 for this. + +## API + +The API is a Python 3.6 app designed to run on AWS Lambda. It can be used with any ActionKit instance by changing the settings to point to your ActionKit database (copy settings.py.template to settings.py). Each individual script (validate_key.py and export_rsvps.py) can also be run from the command line for testing. + +### Deploy + +The API can be deployed using [Zappa](https://github.com/Miserlou/Zappa) with the provided zappa_settings.yml.template (copied to zappa_settings.yml). diff --git a/dev_requirements.txt b/dev_requirements.txt new file mode 100644 index 0000000..35c0b69 --- /dev/null +++ b/dev_requirements.txt @@ -0,0 +1 @@ +zappa==0.50.0 diff --git a/export_rsvps.py b/export_rsvps.py new file mode 100644 index 0000000..8a398bd --- /dev/null +++ b/export_rsvps.py @@ -0,0 +1,84 @@ +import hashlib + +import psycopg2 +import psycopg2.extras +from pywell.entry_points import run_from_cli, run_from_api_gateway + +import validate_key + +DESCRIPTION = 'Download RSVPs.' + +ARG_DEFINITIONS = { + 'DB_HOST': 'Host for PostgreSQL connection.', + 'DB_NAME': 'Database name.', + 'DB_PASS': 'Pass for database connection.', + 'DB_PORT': 'Port for database connection.', + 'DB_SCHEMA': 'Scheam for database query.', + 'DB_USER': 'Username for database connection.', + 'EXTRA_WHERE': 'Anything to add to the WHERE clause in the RSVP query.', + 'KEY': 'Key to validate.', + 'MAX_AGE': 'Number of days a key should be considered valid.', + 'SECRET': 'Secret to use for validation.', +} + +REQUIRED_ARGS = [ + 'DB_HOST', 'DB_NAME', 'DB_PASS', 'DB_PORT', 'DB_USER', 'KEY', 'MAX_AGE', + 'SECRET', +] + + +def main(args): + key = validate_key.main(args) + if key.get('valid', False): + connection = psycopg2.connect( + host=args.DB_HOST, + port=args.DB_PORT, + user=args.DB_USER, + password=args.DB_PASS, + database=args.DB_NAME + ) + cursor = connection.cursor( + cursor_factory=psycopg2.extras.RealDictCursor + ) + query = """ + SELECT + u.email, u.first_name, u.middle_name, u.last_name, u.state, u.city, + u.zip, MIN(a.created_at) AS action_datetime, MAX(s.role) AS role + FROM %s.core_user u + JOIN %s.events_eventsignup s ON s.user_id = u.id + JOIN %s.events_event e ON e.id = s.event_id + JOIN %s.events_campaign c ON c.id = e.campaign_id + LEFT JOIN %s.core_action a ON ( + a.page_id = s.page_id + AND a.user_id = u.id + ) + WHERE c.name = %s + %s + AND a.source = %s + GROUP BY 1,2,3,4,5,6,7""" % ( + args.DB_SCHEMA, + args.DB_SCHEMA, + args.DB_SCHEMA, + args.DB_SCHEMA, + args.DB_SCHEMA, + '%s', + args.EXTRA_WHERE, + '%s' + ) + cursor.execute(query, (key.get('campaign', ''), key.get('source', ''))) + return [dict(row) for row in cursor.fetchall()] + else: + return False + + +def aws_lambda(event, context) -> str: + from datetime import datetime + today = datetime.now().strftime('%Y-%m-%d') + return run_from_api_gateway( + main, DESCRIPTION, ARG_DEFINITIONS, REQUIRED_ARGS, event, + format='CSV', filename='moveon_event_signups-%s.csv' % today + ) + + +if __name__ == '__main__': + run_from_cli(main, DESCRIPTION, ARG_DEFINITIONS, REQUIRED_ARGS) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..97be3ca --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +psycopg2==2.8.4 +psycopg2-binary==2.8.2 +git+https://github.com/sreynen/pywell@main#egg=pywell diff --git a/settings.py.template b/settings.py.template new file mode 100644 index 0000000..8aaee56 --- /dev/null +++ b/settings.py.template @@ -0,0 +1,11 @@ +import os + +EXTRA_WHERE = os.environ.get('EXTRA_WHERE', '') +DB_HOST = os.environ.get('PSQL_DB_HOST', '') +DB_NAME = os.environ.get('PSQL_DB_NAME', '') +DB_PASS = os.environ.get('PSQL_DB_PASS', '') +DB_PORT = os.environ.get('PSQL_DB_PORT', ) +DB_SCHEMA = os.environ.get('PSQL_DB_SCHEMA', '') +DB_USER = os.environ.get('PSQL_DB_USER', '') +SECRET = os.environ.get('SECRET', '') +MAX_AGE = int(os.environ.get('MAX_AGE', 2)) diff --git a/validate_key.py b/validate_key.py new file mode 100644 index 0000000..39349b2 --- /dev/null +++ b/validate_key.py @@ -0,0 +1,56 @@ +from datetime import datetime, timedelta +import hashlib + +from pywell.entry_points import run_from_cli, run_from_api_gateway + + +DESCRIPTION = 'Validate a download key.' + +ARG_DEFINITIONS = { + 'KEY': 'Key to validate.', + 'MAX_AGE': 'Number of days a key should be considered valid.', + 'SECRET': 'Secret to use for validation.' +} + +REQUIRED_ARGS = ['KEY', 'MAX_AGE', 'SECRET'] + + +def main(args): + # If key doesn't have enough parts, it's invalid. + if len(args.KEY.split('.')) != 5: + return {'valid': False} + [key_created, age, source, campaign, hash] = args.KEY.split('.') + m = hashlib.sha256() + m.update( + ( + '%s.%s.%s.%s.%s' % (key_created, age, source, campaign, args.SECRET) + ).encode('utf-8') + ) + hash_check = m.hexdigest() + # If key hash doesn't match prefix, it's invalid. + if hash_check != hash: + return {'valid': False} + # If key age is too big, it's invalid. + if int(age) > args.MAX_AGE: + return {'valid': False} + # If key was created more than age days ago, it's invalid. + key_created_min = (datetime.now() - timedelta(days=int(age))).strftime('%Y-%m-%d') + if key_created_min > key_created: + return {'valid': False} + # Otherwise, it's a valid key. Note: valid doesn't mean it will have any + # results for the source and campaign. We don't check that until export. + return { + 'valid': True, + 'date': key_created, + 'age': age, + 'source': source, + 'campaign': campaign + } + + +def aws_lambda(event, context) -> str: + return run_from_api_gateway(main, DESCRIPTION, ARG_DEFINITIONS, REQUIRED_ARGS, event) + + +if __name__ == '__main__': + run_from_cli(main, DESCRIPTION, ARG_DEFINITIONS, REQUIRED_ARGS) diff --git a/zappa_settings.yml.template b/zappa_settings.yml.template new file mode 100644 index 0000000..b206a43 --- /dev/null +++ b/zappa_settings.yml.template @@ -0,0 +1,37 @@ +--- +validate: + apigateway_enabled: false + # We're using a single API gateway for multiple Lambdas here. Zappa can't + # do this, so the API gateway config is done manually. + aws_region: + keep_warm: false + lambda_handler: validate_key.aws_lambda + memory_size: 2048 + project_name: ak-partner-rsvp + role_name: + runtime: python3.6 + s3_bucket: + timeout_seconds: 300 + vpc_config: + SubnetIds: + - + SecurityGroupIds: + - +export: + apigateway_enabled: false + # We're using a single API gateway for multiple Lambdas here. Zappa can't + # do this, so the API gateway config is done manually. + aws_region: + keep_warm: false + lambda_handler: export_rsvps.aws_lambda + memory_size: 2048 + project_name: ak-partner-rsvp + role_name: + runtime: python3.6 + s3_bucket: + timeout_seconds: 300 + vpc_config: + SubnetIds: + - + SecurityGroupIds: + - From 3f3365c2fddaa631a5cc22a2f4ee6a25d308fb65 Mon Sep 17 00:00:00 2001 From: Scott Reynen Date: Mon, 30 Mar 2020 08:48:38 -0600 Subject: [PATCH 2/3] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 137014d..c789a44 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # ActionKit Partner RSVP Export -This is a tool to allow partners to export their own sourced RSVPs from ActionKit event campaigns. It has two parts: a static site and an API. +This is a tool to allow partners to export their own sourced RSVPs from ActionKit event campaigns. It has two parts: a static site, with HTML and JavaScript that calls an API, and the code for that API. ## Static site -The static site in the repo is MoveOn-specific. If you're using this with a different ActionKit instance, you'd of course want to change the HTML and CSS to match your organization's brand, and also edit the apiRoot value at the top of static/index.js. +The static HTML in the repo is MoveOn-specific. If you're using this with a different ActionKit instance, you'd of course want to change the HTML and CSS to match your organization's brand, and also edit the apiRoot value at the top of static/index.js. ### Deploy From 697882ad739c53482dcee1aee8cb0c48979ad2a6 Mon Sep 17 00:00:00 2001 From: Scott Reynen Date: Fri, 3 Apr 2020 08:04:07 -0600 Subject: [PATCH 3/3] add validate tests --- dev_requirements.txt | 1 + test_validate_key.py | 83 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 test_validate_key.py diff --git a/dev_requirements.txt b/dev_requirements.txt index 35c0b69..6077ebb 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1 +1,2 @@ +pytest==5.0.1 zappa==0.50.0 diff --git a/test_validate_key.py b/test_validate_key.py new file mode 100644 index 0000000..95ea65c --- /dev/null +++ b/test_validate_key.py @@ -0,0 +1,83 @@ +from datetime import datetime, timedelta +import hashlib + +import pytest + +import validate_key + + +class Struct: + def __init__(self, **entries): + self.__dict__.update(entries) + + +class Test: + def test_valid_key(self): + yesterday = (datetime.now() - timedelta(days=int(1))).strftime('%Y-%m-%d') + m = hashlib.sha256() + prefix = '%s.2.asource.acampaign' % yesterday + m.update((prefix + '.secret').encode('utf-8')) + hash = m.hexdigest() + args = { + 'KEY': prefix + '.' + hash, + 'MAX_AGE': 2, + 'SECRET': 'secret' + } + args_struct = Struct(**args) + result = validate_key.main(args_struct) + assert result.get('valid') == True + + def test_bad_format_key(self): + args = { + 'KEY': 'invalidkey', + 'MAX_AGE': 2, + 'SECRET': 'secret' + } + args_struct = Struct(**args) + result = validate_key.main(args_struct) + assert result.get('valid') == False + + def test_old_key(self): + last_week = (datetime.now() - timedelta(days=int(7))).strftime('%Y-%m-%d') + m = hashlib.sha256() + prefix = '%s.2.asource.acampaign' % last_week + m.update((prefix + '.secret').encode('utf-8')) + hash = m.hexdigest() + args = { + 'KEY': prefix + '.' + hash, + 'MAX_AGE': 2, + 'SECRET': 'secret' + } + args_struct = Struct(**args) + result = validate_key.main(args_struct) + assert result.get('valid') == False + + def test_wrong_secret(self): + yesterday = (datetime.now() - timedelta(days=int(1))).strftime('%Y-%m-%d') + m = hashlib.sha256() + prefix = '%s.2.asource.acampaign' % yesterday + m.update((prefix + '.wrongsecret').encode('utf-8')) + hash = m.hexdigest() + args = { + 'KEY': prefix + '.' + hash, + 'MAX_AGE': 2, + 'SECRET': 'secret' + } + args_struct = Struct(**args) + result = validate_key.main(args_struct) + assert result.get('valid') == False + + def test_max_age_too_high(self): + yesterday = (datetime.now() - timedelta(days=int(1))).strftime('%Y-%m-%d') + m = hashlib.sha256() + prefix = '%s.3.asource.acampaign' % yesterday + m.update((prefix + '.secret').encode('utf-8')) + hash = m.hexdigest() + args = { + 'KEY': prefix + '.' + hash, + 'MAX_AGE': 2, + 'SECRET': 'secret' + } + args_struct = Struct(**args) + result = validate_key.main(args_struct) + assert result.get('valid') == False