From 24155210df35600f41f479c8aad21e013d3585ef Mon Sep 17 00:00:00 2001 From: Christopher Merrick Date: Thu, 25 Feb 2016 14:32:58 -0500 Subject: [PATCH 01/88] add basic trello models --- notes.txt => README.md | 10 ++++++-- models/trello/base.sql | 44 +++++++++++++++++++++++++++++++++++ models/trello/transformed.sql | 22 ++++++++++++++++++ 3 files changed, 74 insertions(+), 2 deletions(-) rename notes.txt => README.md (89%) create mode 100644 models/trello/base.sql create mode 100644 models/trello/transformed.sql diff --git a/notes.txt b/README.md similarity index 89% rename from notes.txt rename to README.md index 92dc7d4..1833db4 100644 --- a/notes.txt +++ b/README.md @@ -1,12 +1,18 @@ -sql construction conventions +### analyst-collective/models + +A collection of model definitions for common data sets in SQL + +### contributing + +##### sql construction conventions - first layer should simply set fields and table / schema - second layer should be filter. if no records to be filtered out, simply implement as select *. - third layer should be transformations. this could include datatype conversions, mapping, and other simple transformations to make the data more standardized and consumable. *all transformations must meet the strict definition of universal applicability.* - all files should be DDL (should create permanent database objects, not just execute queries) - all files should contain a single ddl operation +### questions -questions - need to create a destination schema for views created. - should be separate schema for each source system or all together in a single schema? - should scripts automatically drop / recreate schemas? much cleaner but high potential for fuckup by users not paying attention. diff --git a/models/trello/base.sql b/models/trello/base.sql new file mode 100644 index 0000000..1c32c4f --- /dev/null +++ b/models/trello/base.sql @@ -0,0 +1,44 @@ +drop schema if exists trello cascade; +create schema trello; + +create or replace view trello.cards_base as ( + + select + id, + idlist, + idboard, + idshort, + name, + datelastactivity::timestamp as datelastactivity, + due::timestamp as due, + closed + from + trello_growth.trello_cards + +); + +create or replace view trello.lists_base as ( + + select + id, + idboard, + name, + closed + from + trello_growth.trello_lists + +); + +create or replace view trello.actions_base as ( + + select + id, + idmembercreator, + data__board__id as idboard, + data__list__id as idlist, + data__card__id as idcard, + date::timestamp as date, + "type" + from trello_growth.trello_actions + +); diff --git a/models/trello/transformed.sql b/models/trello/transformed.sql new file mode 100644 index 0000000..f3c058d --- /dev/null +++ b/models/trello/transformed.sql @@ -0,0 +1,22 @@ +create or replace view trello.cards as ( + + select + c.id, + c.idlist, + c.idboard, + c.idshort, + c.name, + c.datelastactivity, + c.due, + c.closed, + created_action.date as created_at + from + trello.cards_base c + join + trello.actions_base created_action + on + created_action.cardid = c.id + and created_action.cardid is not null + and created_action.type = 'createCard' + +); From c98e066acfdfb3bb33171485867ebe2cda8e1597 Mon Sep 17 00:00:00 2001 From: Christopher Merrick Date: Thu, 25 Feb 2016 15:53:16 -0500 Subject: [PATCH 02/88] move datatype coersions and field renaming --- models/trello/base.sql | 12 ++++++------ models/trello/transformed.sql | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/models/trello/base.sql b/models/trello/base.sql index 1c32c4f..7305cd3 100644 --- a/models/trello/base.sql +++ b/models/trello/base.sql @@ -9,8 +9,8 @@ create or replace view trello.cards_base as ( idboard, idshort, name, - datelastactivity::timestamp as datelastactivity, - due::timestamp as due, + datelastactivity, + due, closed from trello_growth.trello_cards @@ -34,10 +34,10 @@ create or replace view trello.actions_base as ( select id, idmembercreator, - data__board__id as idboard, - data__list__id as idlist, - data__card__id as idcard, - date::timestamp as date, + data__board__id, + data__list__id, + data__card__id, + date, "type" from trello_growth.trello_actions diff --git a/models/trello/transformed.sql b/models/trello/transformed.sql index f3c058d..ea3b89d 100644 --- a/models/trello/transformed.sql +++ b/models/trello/transformed.sql @@ -2,14 +2,14 @@ create or replace view trello.cards as ( select c.id, - c.idlist, - c.idboard, + c.data__board__id as idboard, + c.data__list__id as idlist, c.idshort, c.name, - c.datelastactivity, - c.due, + c.datelastactivity::timestamp as datelastactivity, + c.due::timestamp as due, c.closed, - created_action.date as created_at + created_action.date::timestamp as created_at from trello.cards_base c join From d89fb4632a9db74f5c7de9175fa7a16caf2a0ddd Mon Sep 17 00:00:00 2001 From: Christopher Merrick Date: Thu, 25 Feb 2016 15:57:45 -0500 Subject: [PATCH 03/88] test fixes --- models/trello/transformed.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/models/trello/transformed.sql b/models/trello/transformed.sql index ea3b89d..86b7844 100644 --- a/models/trello/transformed.sql +++ b/models/trello/transformed.sql @@ -2,8 +2,8 @@ create or replace view trello.cards as ( select c.id, - c.data__board__id as idboard, - c.data__list__id as idlist, + c.idboard, + c.idlist, c.idshort, c.name, c.datelastactivity::timestamp as datelastactivity, @@ -15,8 +15,8 @@ create or replace view trello.cards as ( join trello.actions_base created_action on - created_action.cardid = c.id - and created_action.cardid is not null + created_action.data__card__id = c.id + and created_action.data__card__id is not null and created_action.type = 'createCard' ); From ff29b89330eb5a664bb23ad2fed75854da896ca3 Mon Sep 17 00:00:00 2001 From: Christopher Merrick Date: Fri, 4 Mar 2016 22:06:59 -0500 Subject: [PATCH 04/88] remove trello boilerplate, add card_location model --- models/rjm/pipeline/trello/model.sql | 18 ++++++++ models/trello/base.sql | 44 ------------------- models/trello/transformed.sql | 22 ---------- .../pipeline/trello/card_location_test.sql | 5 +++ 4 files changed, 23 insertions(+), 66 deletions(-) create mode 100644 models/rjm/pipeline/trello/model.sql delete mode 100644 models/trello/base.sql delete mode 100644 models/trello/transformed.sql create mode 100644 test/rjm/pipeline/trello/card_location_test.sql diff --git a/models/rjm/pipeline/trello/model.sql b/models/rjm/pipeline/trello/model.sql new file mode 100644 index 0000000..6606c77 --- /dev/null +++ b/models/rjm/pipeline/trello/model.sql @@ -0,0 +1,18 @@ +create or replace view {schema}.card_location as ( + select + id, + idmembercreator, + date, + "type", + data__card__id, + data__card__name, + coalesce(data__listafter__id, data__list__id) as data__list__id, + coalesce(data__boardtarget__id, data__board__id) as data__board__id, + coalesce(data__card__closed, + lag(data__card__closed) ignore nulls over (partition by data__card__id order by date), + false) as data__card__closed + from trello_growth.trello_actions + where + data__card__id is not null + and "type" in ('createCard', 'updateCard', 'moveCardFromBoard', 'moveCardToBoard') +); diff --git a/models/trello/base.sql b/models/trello/base.sql deleted file mode 100644 index 7305cd3..0000000 --- a/models/trello/base.sql +++ /dev/null @@ -1,44 +0,0 @@ -drop schema if exists trello cascade; -create schema trello; - -create or replace view trello.cards_base as ( - - select - id, - idlist, - idboard, - idshort, - name, - datelastactivity, - due, - closed - from - trello_growth.trello_cards - -); - -create or replace view trello.lists_base as ( - - select - id, - idboard, - name, - closed - from - trello_growth.trello_lists - -); - -create or replace view trello.actions_base as ( - - select - id, - idmembercreator, - data__board__id, - data__list__id, - data__card__id, - date, - "type" - from trello_growth.trello_actions - -); diff --git a/models/trello/transformed.sql b/models/trello/transformed.sql deleted file mode 100644 index 86b7844..0000000 --- a/models/trello/transformed.sql +++ /dev/null @@ -1,22 +0,0 @@ -create or replace view trello.cards as ( - - select - c.id, - c.idboard, - c.idlist, - c.idshort, - c.name, - c.datelastactivity::timestamp as datelastactivity, - c.due::timestamp as due, - c.closed, - created_action.date::timestamp as created_at - from - trello.cards_base c - join - trello.actions_base created_action - on - created_action.data__card__id = c.id - and created_action.data__card__id is not null - and created_action.type = 'createCard' - -); diff --git a/test/rjm/pipeline/trello/card_location_test.sql b/test/rjm/pipeline/trello/card_location_test.sql new file mode 100644 index 0000000..90f2731 --- /dev/null +++ b/test/rjm/pipeline/trello/card_location_test.sql @@ -0,0 +1,5 @@ +select id + from card_locations +where + data_card_closed = true + and data__list__id is not null; -- assertEmpty From 9045346e1ff9734eeabe2353cec8fb0a0605ca40 Mon Sep 17 00:00:00 2001 From: Christopher Merrick Date: Fri, 4 Mar 2016 22:09:05 -0500 Subject: [PATCH 05/88] fix whitespace --- models/rjm/pipeline/trello/model.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/rjm/pipeline/trello/model.sql b/models/rjm/pipeline/trello/model.sql index 6606c77..cb6d1c9 100644 --- a/models/rjm/pipeline/trello/model.sql +++ b/models/rjm/pipeline/trello/model.sql @@ -1,7 +1,7 @@ create or replace view {schema}.card_location as ( select id, - idmembercreator, + idmembercreator, date, "type", data__card__id, From fba2613659fd50e64a0b8a0b263202dd62fda140 Mon Sep 17 00:00:00 2001 From: Christopher Merrick Date: Sat, 5 Mar 2016 00:25:30 -0500 Subject: [PATCH 06/88] trello card location model --- config.json | 4 ++-- models/{rjm/pipeline => }/trello/model.sql | 6 ++++-- test/rjm/pipeline/trello/card_location_test.sql | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) rename models/{rjm/pipeline => }/trello/model.sql (61%) diff --git a/config.json b/config.json index dc5b69e..2042e57 100644 --- a/config.json +++ b/config.json @@ -1,4 +1,4 @@ { - "models" : ["pardot", "segment", "snowplow"], - "schema" : "analyst_collective" + "models" : ["pardot", "segment", "snowplow", "trello"], + "schema" : "analyst_collective" } diff --git a/models/rjm/pipeline/trello/model.sql b/models/trello/model.sql similarity index 61% rename from models/rjm/pipeline/trello/model.sql rename to models/trello/model.sql index cb6d1c9..ef868b0 100644 --- a/models/rjm/pipeline/trello/model.sql +++ b/models/trello/model.sql @@ -6,10 +6,12 @@ create or replace view {schema}.card_location as ( "type", data__card__id, data__card__name, - coalesce(data__listafter__id, data__list__id) as data__list__id, + coalesce(data__list__id, + data__listafter__id, + lag(coalesce(data__list__id, data__listafter__id)) ignore nulls over (partition by data__card__id order by date)) as data__list__id, coalesce(data__boardtarget__id, data__board__id) as data__board__id, coalesce(data__card__closed, - lag(data__card__closed) ignore nulls over (partition by data__card__id order by date), + lag(data__card__closed) ignore nulls over (partition by data__card__id order by date), false) as data__card__closed from trello_growth.trello_actions where diff --git a/test/rjm/pipeline/trello/card_location_test.sql b/test/rjm/pipeline/trello/card_location_test.sql index 90f2731..81fe320 100644 --- a/test/rjm/pipeline/trello/card_location_test.sql +++ b/test/rjm/pipeline/trello/card_location_test.sql @@ -1,5 +1,5 @@ select id - from card_locations + from analyst_collective.trello_card_location where - data_card_closed = true + data__card__closed = true and data__list__id is not null; -- assertEmpty From 72c51f43a5e50ff4bd21b23aad02591b65d611ce Mon Sep 17 00:00:00 2001 From: Christopher Merrick Date: Sat, 5 Mar 2016 14:25:12 -0500 Subject: [PATCH 07/88] use commentCard actions for location updates --- models/trello/model.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/trello/model.sql b/models/trello/model.sql index ef868b0..d7ac3b7 100644 --- a/models/trello/model.sql +++ b/models/trello/model.sql @@ -16,5 +16,5 @@ create or replace view {schema}.card_location as ( from trello_growth.trello_actions where data__card__id is not null - and "type" in ('createCard', 'updateCard', 'moveCardFromBoard', 'moveCardToBoard') + and "type" in ('createCard', 'updateCard', 'moveCardFromBoard', 'moveCardToBoard', 'commentCard') ); From 1314047a8cdcc38e912cda0b3c9641f4ab01a758 Mon Sep 17 00:00:00 2001 From: Christopher Merrick Date: Sat, 5 Mar 2016 14:25:50 -0500 Subject: [PATCH 08/88] change trello test namespace --- test/{rjm/pipeline => }/trello/card_location_test.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{rjm/pipeline => }/trello/card_location_test.sql (100%) diff --git a/test/rjm/pipeline/trello/card_location_test.sql b/test/trello/card_location_test.sql similarity index 100% rename from test/rjm/pipeline/trello/card_location_test.sql rename to test/trello/card_location_test.sql From 251f4158120ceeef7adc39d1e05ec0a3b85ab752 Mon Sep 17 00:00:00 2001 From: Christopher Merrick Date: Sat, 5 Mar 2016 16:10:09 -0500 Subject: [PATCH 09/88] add trello model tests and have runner handle all .sql files --- models/trello/test.sql | 25 +++++++++++++++ scripts/analyst_collective/runner.py | 48 ++++++++++++++++++++-------- test/trello/card_location_test.sql | 5 --- 3 files changed, 59 insertions(+), 19 deletions(-) create mode 100644 models/trello/test.sql delete mode 100644 test/trello/card_location_test.sql diff --git a/models/trello/test.sql b/models/trello/test.sql new file mode 100644 index 0000000..0b684fc --- /dev/null +++ b/models/trello/test.sql @@ -0,0 +1,25 @@ +create or replace view {schema}.model_tests +(name, description, result) +as ( + with null_boards_or_lists as + ( + select id + from analyst_collective.trello_card_location + where + data__board__id is null + or data__list__id is null + ) + select + 'no_null_boards_or_lists', + 'All location entries have a non-null board and list id', + count(*) = 0 + from null_boards_or_lists + + union all + + select + 'fresher_than_one_day', + 'Most recent entry is no more than one day old', + max(date::timestamp) > current_date - '1 day'::interval + from analyst_collective.trello_card_location +); diff --git a/scripts/analyst_collective/runner.py b/scripts/analyst_collective/runner.py index 7941e45..aaba9ec 100644 --- a/scripts/analyst_collective/runner.py +++ b/scripts/analyst_collective/runner.py @@ -1,5 +1,5 @@ -import sqlparse, psycopg2, sys, os +import sqlparse, psycopg2, sys, os, fnmatch class Runner(object): def __init__(self, config, creds, models_dir): @@ -10,7 +10,7 @@ def __init__(self, config, creds, models_dir): self.connection = psycopg2.connect(creds.conn_string) def models(self): - return self.config['models'] + return set(self.config['models']) def drop_schema(self): sql = self.interpolate("drop schema if exists {schema} cascade") @@ -44,21 +44,41 @@ def add_prefix(self, uninterpolated_sql, model): replace = "{schema}.{model}_" return uninterpolated_sql.replace(match, replace) + def __model_files(self): + """returns a dictionary like +{'pardot': ['pardot/model.sql'], + 'segment': ['segment/model.sql'], + 'snowplow': ['snowplow/model.sql'], + 'trello': ['trello/model.sql', 'trello/test.sql']} +""" + indexed_files = {} + for root, dirs, files in os.walk(self.models_dir): + for filename in files: + if fnmatch.fnmatch(filename, "*.sql"): + abs_path = os.path.join(root, filename) + rel_path = os.path.relpath(abs_path, self.models_dir) + namespace = os.path.dirname(rel_path).replace('/', '.') + indexed_files.setdefault(namespace, []).append(rel_path) + return indexed_files + def create_models(self): - for model_name in self.models(): - # right now, this only checks for model.sql in the model dir. It can ideally load the SQL file DAG - model_file = os.path.join(self.models_dir, model_name, 'model.sql') + for namespace, files in self.__model_files().iteritems(): + if namespace not in self.models(): + continue - contents = None - with open(model_file) as model_fh: - contents = model_fh.read() + for f in sorted(files): + model_file = os.path.join(self.models_dir, f) - statements = sqlparse.parse(contents); - for statement in statements: - prefixed = self.add_prefix(str(statement), model_name) - sql = self.interpolate(prefixed, model_name) + contents = None + with open(model_file) as model_fh: + contents = model_fh.read() - if sql is None or len(sql.strip()) == 0: continue # could throw an error here! Definitely don't execute the sql though + statements = sqlparse.parse(contents) + for statement in statements: + prefixed = self.add_prefix(str(statement), namespace) + sql = self.interpolate(prefixed, namespace) - self.execute(sql) + if sql is None or len(sql.strip()) == 0: + continue # could throw an error here! Definitely don't execute the sql though + self.execute(sql) diff --git a/test/trello/card_location_test.sql b/test/trello/card_location_test.sql deleted file mode 100644 index 81fe320..0000000 --- a/test/trello/card_location_test.sql +++ /dev/null @@ -1,5 +0,0 @@ -select id - from analyst_collective.trello_card_location -where - data__card__closed = true - and data__list__id is not null; -- assertEmpty From 6927f01bb667c070e1e0dedf290624ab0a9ff111 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Tue, 8 Mar 2016 09:13:09 -0500 Subject: [PATCH 10/88] basic email interface model for pardot --- analysis/email/interface.txt | 22 ++++++++++++++++++++++ models/pardot/model.sql | 29 ++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 analysis/email/interface.txt diff --git a/analysis/email/interface.txt b/analysis/email/interface.txt new file mode 100644 index 0000000..dea0bed --- /dev/null +++ b/analysis/email/interface.txt @@ -0,0 +1,22 @@ +######################################## +# Email Interface # +######################################## + +timestamp timestamp +event varchar +user_id varchar +email_id varchar +details varchar + + +The following events are need for the basic email interface: +- send +- open +- click + +The following events are need for the extended email interface: +- hard bounce +- soft bounce +- subscribe +- unsubscribe +- spam complaint diff --git a/models/pardot/model.sql b/models/pardot/model.sql index c369f02..617d8af 100644 --- a/models/pardot/model.sql +++ b/models/pardot/model.sql @@ -46,14 +46,14 @@ create or replace view {schema}.visitoractivity_events_meta as ( --even with the type decoding that Pardot specifically provides, actually what is going on in a given event --is somewhat ambiguous. this is an attempt to map type and type_name to a more event-based "event action" field - --which is always written in more standard action-oriented terms. + --which is always written in more standard action-oriented terms. select 22 as "type", 'Chat Transcript' as type_name, 'chatted via olark' as event_name union all select 21, 'Custom Redirect', 'clicked a custom redirect' union all - select 6, 'Email', 'sent an email' union all - select 11, 'Email', 'opened an email' union all - select 13, 'Email', 'bounced email' union all - select 14, 'Email', 'reported spam' union all - select 1, 'Email Tracker', 'clicked on email link' union all + select 6, 'Email', 'email sent' union all + select 11, 'Email', 'email opened' union all + select 13, 'Email', 'email bounced' union all + select 14, 'Email', 'email reported spam' union all + select 1, 'Email Tracker', 'email click' union all select 28, 'Event', 'registered for event' union all select 29, 'Event', 'checked in at event' union all select 2, 'File', 'viewed a file' union all @@ -90,7 +90,7 @@ create or replace view {schema}.visitoractivity as ( select -- event_stream interface va.created_at as "@timestamp", - t.type_decoded as "@event", + e.event_name as "@event", va.prospect_id as "@user_id", va.* from @@ -102,3 +102,18 @@ create or replace view {schema}.visitoractivity as ( ); COMMENT ON VIEW {schema}.visitoractivity IS 'timeseries,funnel,cohort'; + + +/* +This model maps pardot data from the visitoractivity table to the email analysis interface. +It conforms to the basic email interface, not the extended email interface, because Pardot does not supply +data necessary to conform to the extended interface. +*/ + +create or replace view {schema}.email as ( + + select "@timestamp", "@event", "@user_id", email_id as "@email_id", details as "@subject" + from {schema}.visitoractivity + where "@event" in ('email sent', 'email opened', 'email click') + +); From 3c7f28277fbfb7e19e55267b99b446801c41a2f8 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Wed, 9 Mar 2016 17:42:28 -0500 Subject: [PATCH 11/88] lots of work on email analysis --- analysis/email/analysis.sql | 114 ++++++++++++++++++++++++++++++++++++ analysis/email/model.sql | 41 +++++++++++++ models/pardot/model.sql | 2 +- models/pardot/tests.sql | 26 ++++++++ 4 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 analysis/email/analysis.sql create mode 100644 analysis/email/model.sql create mode 100644 models/pardot/tests.sql diff --git a/analysis/email/analysis.sql b/analysis/email/analysis.sql new file mode 100644 index 0000000..949cdce --- /dev/null +++ b/analysis/email/analysis.sql @@ -0,0 +1,114 @@ +/* + +This analysis gets you a simple funnel of sent > opened > clicked emails for all-time. + +*/ + +with events as ( + + select * from analyst_collective.pardot_email + +), + +funnel_order as ( + + select 'email sent' as event, 1 as "@order" union all + select 'email opened', 2 union all + select 'email click', 3 + +) + +--cr stands for conversion rate + select e."@event", count(*), + count(*)::float / lag(count(*)) over (order by fo."@order")::float as cr, + count(distinct e."@user_id") as distinct_users, + count(distinct e."@user_id")::float / lag(count(distinct e."@user_id")) over (order by fo."@order")::float as distinct_cr + from events e + inner join funnel_order fo on e."@event" = fo.event + group by 1, fo."@order" + order by fo."@order" + ; + + + + +/* + +This analysis gets you a simple timeseries of sent > opened > clicked emails for all-time. + +*/ + + + with events as ( + + select * from analyst_collective.pardot_email + + ) + + select "@event", date_trunc('week', "@timestamp"), count(*) + from events + group by 1, 2; + + + +/* + +Open/click rate by user's email number. Email number 1 is the first email sent to that user, etc. +Do people stop engaging with emails the more emails they have been sent? + +*/ + + +with emails as ( + + select * from analyst_collective.emails_denormalized + +) + +select email_number, avg("opened?"::float) as open_rate, avg("clicked?"::float) as ctr, + count(*) as num_users +from emails +--only look at the first 25 emails someone is sent. should be customized based on business. +where email_number < 26 +group by 1 +order by 1 +; + + + + +/* + +Open/Click rate by email frequency. +Does email sending volume have an impact on engagement with emails sent? +In the extended email interface, need to do the same thing for unsubscribes, because this is where we have seen the strong correlation in the past. +This interface doesn't contain unsubscribe data. + +*/ + + +with emails as ( + + select * from analyst_collective.emails_denormalized + +) + +select date_trunc('month', sent_timestamp), + count(*)::float / count(distinct "@user_id")::float as emails_per_user, + avg("opened?"::float) as open_rate, avg("clicked?"::float) as ctr +from emails +group by 1 +order by 1 +; + + + + +/* + +Other analysis I want to do: + - cohort analysis of email engagement vs first email send date. + - anomaly detection for email performance: particularly good or bad email subject lines by open rate. leave out bottom 20% of send volume. + - likelihood of opening / clicking by length (time or # of emails) of user inactivity + +*/ diff --git a/analysis/email/model.sql b/analysis/email/model.sql new file mode 100644 index 0000000..e22cc7a --- /dev/null +++ b/analysis/email/model.sql @@ -0,0 +1,41 @@ +create or replace view {schema}.emails_denormalized as ( + + with events as ( + + select * from {schema}.emails + + ), + + sends as ( + + select "@user_id", "@timestamp", "@email_id" + from events + where "@event" = 'email sent' + + ), + + opens as ( + + select "@user_id", "@timestamp", "@email_id" + from events + where "@event" = 'email opened' + + ), clicks as ( + + select "@user_id", "@timestamp", "@email_id" + from events + where "@event" = 'email click' + + ), emails as ( + + select s."@user_id", s."@timestamp" as sent_timestamp, o."@timestamp" as opened_timestamp, c."@timestamp" as clicked_timestamp, + decode(o."@timestamp", null, 0, 1) as "opened?", + decode(c."@timestamp", null, 0, 1) as "clicked?", + row_number() over (partition by s."@user_id" order by s."@timestamp") as email_number + from sends s + left outer join opens o on s."@email_id" = o."@email_id" + left outer join clicks c on s."@email_id" = c."@email_id" + order by 1, 2 + + ) +); diff --git a/models/pardot/model.sql b/models/pardot/model.sql index 617d8af..b3a0981 100644 --- a/models/pardot/model.sql +++ b/models/pardot/model.sql @@ -110,7 +110,7 @@ It conforms to the basic email interface, not the extended email interface, beca data necessary to conform to the extended interface. */ -create or replace view {schema}.email as ( +create or replace view {schema}.emails as ( select "@timestamp", "@event", "@user_id", email_id as "@email_id", details as "@subject" from {schema}.visitoractivity diff --git a/models/pardot/tests.sql b/models/pardot/tests.sql new file mode 100644 index 0000000..723a2b3 --- /dev/null +++ b/models/pardot/tests.sql @@ -0,0 +1,26 @@ +create or replace view {schema}.model_tests + (name, description, result) + as ( + + select + 'visitoractivity_fresher_than_one_day', + 'Most recent visitoractivity entry is no more than one day old', + max("@timestamp"::timestamp) > current_date - '1 day'::interval + from {schema}.visitoractivity + + ); + + + + + + + + /* + +Other tests I want to do: + - make sure there are records from every day since the first day we see any records + - make sure all prospect ids from visitoractivity show up in prospects + - make sure there are no unmapped types + + */ From f754016a967ff109daae93c3f89c0d63cd84e1cb Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Wed, 9 Mar 2016 18:30:55 -0500 Subject: [PATCH 12/88] Updated email analysis. --- analysis/email/analysis.sql | 56 ++++++++++--------------------------- analysis/email/model.sql | 3 +- config.json | 4 --- 3 files changed, 16 insertions(+), 47 deletions(-) delete mode 100644 config.json diff --git a/analysis/email/analysis.sql b/analysis/email/analysis.sql index 949cdce..b50f0e0 100644 --- a/analysis/email/analysis.sql +++ b/analysis/email/analysis.sql @@ -1,37 +1,3 @@ -/* - -This analysis gets you a simple funnel of sent > opened > clicked emails for all-time. - -*/ - -with events as ( - - select * from analyst_collective.pardot_email - -), - -funnel_order as ( - - select 'email sent' as event, 1 as "@order" union all - select 'email opened', 2 union all - select 'email click', 3 - -) - ---cr stands for conversion rate - select e."@event", count(*), - count(*)::float / lag(count(*)) over (order by fo."@order")::float as cr, - count(distinct e."@user_id") as distinct_users, - count(distinct e."@user_id")::float / lag(count(distinct e."@user_id")) over (order by fo."@order")::float as distinct_cr - from events e - inner join funnel_order fo on e."@event" = fo.event - group by 1, fo."@order" - order by fo."@order" - ; - - - - /* This analysis gets you a simple timeseries of sent > opened > clicked emails for all-time. @@ -39,15 +5,23 @@ This analysis gets you a simple timeseries of sent > opened > clicked emails for */ - with events as ( +with events as ( - select * from analyst_collective.pardot_email + select * from analyst_collective.emails_denormalized - ) +) - select "@event", date_trunc('week', "@timestamp"), count(*) - from events - group by 1, 2; +select + date_trunc('month', "sent_timestamp") as mnth, + count(*) as sends, + sum("opened?") as opens, + sum("clicked?") as clicks, + avg("opened?"::float) as open_rate, + avg("clicked?"::float) as click_rate +from events +group by 1 +order by 1 +; @@ -106,7 +80,7 @@ order by 1 /* -Other analysis I want to do: +Other analysis I still want to do: - cohort analysis of email engagement vs first email send date. - anomaly detection for email performance: particularly good or bad email subject lines by open rate. leave out bottom 20% of send volume. - likelihood of opening / clicking by length (time or # of emails) of user inactivity diff --git a/analysis/email/model.sql b/analysis/email/model.sql index e22cc7a..c3e260b 100644 --- a/analysis/email/model.sql +++ b/analysis/email/model.sql @@ -26,7 +26,7 @@ create or replace view {schema}.emails_denormalized as ( from events where "@event" = 'email click' - ), emails as ( + ) select s."@user_id", s."@timestamp" as sent_timestamp, o."@timestamp" as opened_timestamp, c."@timestamp" as clicked_timestamp, decode(o."@timestamp", null, 0, 1) as "opened?", @@ -37,5 +37,4 @@ create or replace view {schema}.emails_denormalized as ( left outer join clicks c on s."@email_id" = c."@email_id" order by 1, 2 - ) ); diff --git a/config.json b/config.json deleted file mode 100644 index 2042e57..0000000 --- a/config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "models" : ["pardot", "segment", "snowplow", "trello"], - "schema" : "analyst_collective" -} From ccdacc889d280b04fa0a9e630d5c6b979cbaec1a Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Thu, 10 Mar 2016 14:18:44 -0500 Subject: [PATCH 13/88] drop and create schema in first step of runner --- scripts/analyst_collective/runner.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/analyst_collective/runner.py b/scripts/analyst_collective/runner.py index 0ae6353..5930e12 100644 --- a/scripts/analyst_collective/runner.py +++ b/scripts/analyst_collective/runner.py @@ -12,12 +12,17 @@ def __init__(self, config, creds, models_dir): def models(self): return set(self.config['models']) - def try_create_schema(self): + def drop_schema(self): + sql = self.interpolate("drop schema if exists {schema} cascade") + self.execute(sql) + + def create_schema(self): sql = self.interpolate("create schema if not exists {schema}") self.execute(sql) def clean_schema(self): - self.try_create_schema() + self.drop_schema() + self.create_schema() def execute(self, sql): debug = sql.replace("\n", " ").strip()[0:200] From fa1afd9657471d45c2eb0375b43d8f82cf8cc9ac Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Thu, 10 Mar 2016 21:04:28 -0500 Subject: [PATCH 14/88] throw error if schema already exists --- scripts/analyst_collective/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/analyst_collective/runner.py b/scripts/analyst_collective/runner.py index 5930e12..f98362d 100644 --- a/scripts/analyst_collective/runner.py +++ b/scripts/analyst_collective/runner.py @@ -17,7 +17,7 @@ def drop_schema(self): self.execute(sql) def create_schema(self): - sql = self.interpolate("create schema if not exists {schema}") + sql = self.interpolate("create schema {schema}") self.execute(sql) def clean_schema(self): From fd5d6becb14cd3acf68ff7bfade1e5386d7ebf27 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Tue, 15 Mar 2016 18:01:04 -0400 Subject: [PATCH 15/88] ignore config --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 53676d1..e859da0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ dbcredentials.txt env *.pyc +config.json From 98445a3c1ae9c2e66adfc74381c330c0730e495a Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Wed, 16 Mar 2016 13:12:32 -0400 Subject: [PATCH 16/88] zuora models v1 --- analysis/mrr/active_mrr.sql | 59 ++++++++++++++ analysis/mrr/total_mrr_by_month.sql | 114 ++++++++++++++++++++++++++++ config.json | 4 +- config_old.json | 4 + models/zuora/model.sql | 64 ++++++++++++++++ 5 files changed, 243 insertions(+), 2 deletions(-) create mode 100644 analysis/mrr/active_mrr.sql create mode 100644 analysis/mrr/total_mrr_by_month.sql create mode 100644 config_old.json create mode 100644 models/zuora/model.sql diff --git a/analysis/mrr/active_mrr.sql b/analysis/mrr/active_mrr.sql new file mode 100644 index 0000000..437fe96 --- /dev/null +++ b/analysis/mrr/active_mrr.sql @@ -0,0 +1,59 @@ +-- get all subscriptions with possible ammendments for all accounts +with subscr_w_amendments as +( + select + account_number, acc.account_id, sub.subscr_id, + subscr_name, subscr_status, subscr_term_type, + subscr_start, subscr_end, subscr_version, amend_id, amend_start + from ac_yevgeniy.zuora_account acc + inner join ac_yevgeniy.zuora_subscription sub + on acc.account_id = sub.account_id + -- add ammendments + left outer join ac_yevgeniy.zuora_amendment amend + on sub.subscr_id = amend.subscr_id +), + +-- get all rate plan charges +rate_plan_charges as +( + select + account_number, account_id, sub.subscr_id, + subscr_name, subscr_status, subscr_term_type, + subscr_start, subscr_end, subscr_version, amend_id, amend_start, + rpc_start, rpc_end, rpc_last_segment, + min(subscr_start) over() as first_subscr, + "@mrr" as mrr + from subscr_w_amendments sub + inner join ac_yevgeniy.zuora_rate_plan rp + on rp.subscr_id = sub.subscr_id + inner join ac_yevgeniy.zuora_rate_plan_charge rpc + on rpc.rate_plan_id = rp.rate_plan_id +), + + +charges_for_active_plans as +( + select * + from rate_plan_charges + where + -- make sure the subscription is active + subscr_status = 'Active' + and + ( + -- make sure the rate plan charge is current + ( + rpc_start <= current_date + and + rpc_end >= current_date + ) + or subscr_term_type = 'EVERGREEN' + ) + and rpc_last_segment = TRUE +) + + +-- get the active mrr per account +select account_number, round(sum(mrr),2) as active_mrr +from charges_for_active_plans +group by account_number + diff --git a/analysis/mrr/total_mrr_by_month.sql b/analysis/mrr/total_mrr_by_month.sql new file mode 100644 index 0000000..24a265a --- /dev/null +++ b/analysis/mrr/total_mrr_by_month.sql @@ -0,0 +1,114 @@ +-- get all subscriptions with possible ammendments for all accounts +with subscr_w_amendments as +( + select + account_number, acc.account_id, sub.subscr_id, + subscr_name, subscr_status, subscr_term_type, + subscr_start, subscr_end, subscr_version, amend_id, amend_start + from ac_yevgeniy.zuora_account acc + inner join ac_yevgeniy.zuora_subscription sub + on acc.account_id = sub.account_id + -- add ammendments + left outer join ac_yevgeniy.zuora_amendment amend + on sub.subscr_id = amend.subscr_id +), + +-- get all rate plan charges +rate_plan_charges as +( + select + account_number, account_id, sub.subscr_id, + subscr_name, subscr_status, subscr_term_type, + subscr_start, subscr_end, subscr_version, amend_id, amend_start, + rpc_start, rpc_end, rpc_last_segment, + min(subscr_start) over() as first_subscr, + "@mrr" as mrr + from subscr_w_amendments sub + inner join ac_yevgeniy.zuora_rate_plan rp + on rp.subscr_id = sub.subscr_id + inner join ac_yevgeniy.zuora_rate_plan_charge rpc + on rpc.rate_plan_id = rp.rate_plan_id +), + +-- genereate calendar dates, starting with the first subscription date +dates as +( + select date_day, date_trunc('month',date_day)::date as date_month + from + ( + select (first_subscr + row_number() over (order by true))::date as date_day + from rate_plan_charges + ) + where date_day <= current_date +), + + +-- get all charges up to each date in the calendar +charges_up_to_each_date as +( + select + date_day, date_month, account_number, mrr, rpc_start, rpc_end, rpc_last_segment, + amend_start, amend_id, subscr_term_type, subscr_start, subscr_end, subscr_id, + subscr_name, subscr_version, + dateadd(month,1,date_month) as date_month_plus_one, + max(date_day) over (partition by subscr_name, dateadd(month,1,date_month)) as max_subscr_trunc_date, + max(subscr_version) over (partition by subscr_name, date_month) as max_subscr_version_within_date + from dates a + left join rate_plan_charges b + on 1=1 + and rpc_start <= date_day + and rpc_last_segment = 'TRUE' +), + +all_charges_by_month as +( + select date_month, mrr + from charges_up_to_each_date + where + ( + -- make sure the subscriptions are EVERGREEN/falling into an appropriate bucket + ( + ( + rpc_start <= dateadd(month, 1, date_month) + and + rpc_end >= dateadd(month, 1, date_month) + ) + and + ( + amend_start > dateadd(month, 1, date_month) + or + amend_start is null + ) + ) + or + ( + subscr_term_type = 'EVERGREEN' + and subscr_start <= dateadd(month, 1, date_month) + and + ( + subscr_end is null + or + subscr_end >= dateadd(month, 1, date_month) + ) + and + ( + amend_start > dateadd(month, 1, date_month) + or + amend_start is null + ) + ) + ) + and date_day = max_subscr_trunc_date + and subscr_version = max_subscr_version_within_date + and dateadd(month, 1, date_month) <= current_date +) + + + +-- get mrr for each month +select date_month, sum(mrr) as total_mrr +from all_charges_by_month +group by date_month +order by date_month + + diff --git a/config.json b/config.json index 2042e57..facdc22 100644 --- a/config.json +++ b/config.json @@ -1,4 +1,4 @@ { - "models" : ["pardot", "segment", "snowplow", "trello"], - "schema" : "analyst_collective" + "models" : ["zuora"], + "schema" : "ac_yevgeniy" } diff --git a/config_old.json b/config_old.json new file mode 100644 index 0000000..16f3bf4 --- /dev/null +++ b/config_old.json @@ -0,0 +1,4 @@ +{ + "models" : ["pardot", "segment", "snowplow", "trello", "zuora"], + "schema" : "ac_yevgeniy" +} diff --git a/models/zuora/model.sql b/models/zuora/model.sql new file mode 100644 index 0000000..f3d941a --- /dev/null +++ b/models/zuora/model.sql @@ -0,0 +1,64 @@ +create or replace view {schema}.account as +( + select + id as account_id, + accountnumber as account_number, + * + from zuora.zuora_account +); + + +create or replace view {schema}.subscription as +( + select + id as subscr_id, + status as subscr_status, + termtype as subscr_term_type, + accountid as account_id, + contracteffectivedate::timestamp as subscr_start, + subscriptionenddate::timestamp as subscr_end, + name as subscr_name, + "version#392c30e6081c24fb78ddf6d622de4f33"::integer + as subscr_version, + * + from zuora.zuora_subscription +); + +create or replace view {schema}.rate_plan as +( + select + id as rate_plan_id, + subscriptionid as subscr_id, + * + from zuora.zuora_rate_plan +); + + +create or replace view {schema}.rate_plan_charge as +( + select + rateplanid as rate_plan_id, + effectivestartdate::timestamp as rpc_start, + effectiveenddate::timestamp as rpc_end, + mrr as "@mrr", + islastsegment as rpc_last_segment, + * + from zuora.zuora_rate_plan_charge +); + + +create or replace view {schema}.amendment as +( + select + id as amend_id, + subscriptionid as subscr_id, + effectivedate::timestamp as amend_start, + * + from zuora.zuora_amendment +); + + + + + + From 958fdd22fef692f53f54c59aaf6ee8884efb4391 Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Wed, 16 Mar 2016 13:17:07 -0400 Subject: [PATCH 17/88] zuora models v1 --- .gitignore | 1 + config_old.json | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 config_old.json diff --git a/.gitignore b/.gitignore index 53676d1..8c0375c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ dbcredentials.txt env *.pyc +config.json \ No newline at end of file diff --git a/config_old.json b/config_old.json deleted file mode 100644 index 16f3bf4..0000000 --- a/config_old.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "models" : ["pardot", "segment", "snowplow", "trello", "zuora"], - "schema" : "ac_yevgeniy" -} From eafbbb053e896394d0874c77b4f2e11a5b9caf16 Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Wed, 16 Mar 2016 13:17:37 -0400 Subject: [PATCH 18/88] zuora models v1 --- config.json | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 config.json diff --git a/config.json b/config.json deleted file mode 100644 index facdc22..0000000 --- a/config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "models" : ["zuora"], - "schema" : "ac_yevgeniy" -} From 97d3ea22602968ab042dc9783234298597c3eb88 Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Wed, 16 Mar 2016 18:08:40 -0400 Subject: [PATCH 19/88] zuora v2 --- analysis/mrr/active_mrr.sql | 35 +----------------------- analysis/mrr/total_mrr_by_month.sql | 36 +++---------------------- dbt_project.yml | 7 +++++ models/zuora/model.sql | 41 +++++++++++++++++++++++++---- 4 files changed, 47 insertions(+), 72 deletions(-) create mode 100644 dbt_project.yml diff --git a/analysis/mrr/active_mrr.sql b/analysis/mrr/active_mrr.sql index 437fe96..ff5d5a7 100644 --- a/analysis/mrr/active_mrr.sql +++ b/analysis/mrr/active_mrr.sql @@ -1,40 +1,7 @@ --- get all subscriptions with possible ammendments for all accounts -with subscr_w_amendments as -( - select - account_number, acc.account_id, sub.subscr_id, - subscr_name, subscr_status, subscr_term_type, - subscr_start, subscr_end, subscr_version, amend_id, amend_start - from ac_yevgeniy.zuora_account acc - inner join ac_yevgeniy.zuora_subscription sub - on acc.account_id = sub.account_id - -- add ammendments - left outer join ac_yevgeniy.zuora_amendment amend - on sub.subscr_id = amend.subscr_id -), - --- get all rate plan charges -rate_plan_charges as -( - select - account_number, account_id, sub.subscr_id, - subscr_name, subscr_status, subscr_term_type, - subscr_start, subscr_end, subscr_version, amend_id, amend_start, - rpc_start, rpc_end, rpc_last_segment, - min(subscr_start) over() as first_subscr, - "@mrr" as mrr - from subscr_w_amendments sub - inner join ac_yevgeniy.zuora_rate_plan rp - on rp.subscr_id = sub.subscr_id - inner join ac_yevgeniy.zuora_rate_plan_charge rpc - on rpc.rate_plan_id = rp.rate_plan_id -), - - charges_for_active_plans as ( select * - from rate_plan_charges + from ac_yevgeniy.rate_plan_charges where -- make sure the subscription is active subscr_status = 'Active' diff --git a/analysis/mrr/total_mrr_by_month.sql b/analysis/mrr/total_mrr_by_month.sql index 24a265a..a8fcc7c 100644 --- a/analysis/mrr/total_mrr_by_month.sql +++ b/analysis/mrr/total_mrr_by_month.sql @@ -1,34 +1,4 @@ --- get all subscriptions with possible ammendments for all accounts -with subscr_w_amendments as -( - select - account_number, acc.account_id, sub.subscr_id, - subscr_name, subscr_status, subscr_term_type, - subscr_start, subscr_end, subscr_version, amend_id, amend_start - from ac_yevgeniy.zuora_account acc - inner join ac_yevgeniy.zuora_subscription sub - on acc.account_id = sub.account_id - -- add ammendments - left outer join ac_yevgeniy.zuora_amendment amend - on sub.subscr_id = amend.subscr_id -), - --- get all rate plan charges -rate_plan_charges as -( - select - account_number, account_id, sub.subscr_id, - subscr_name, subscr_status, subscr_term_type, - subscr_start, subscr_end, subscr_version, amend_id, amend_start, - rpc_start, rpc_end, rpc_last_segment, - min(subscr_start) over() as first_subscr, - "@mrr" as mrr - from subscr_w_amendments sub - inner join ac_yevgeniy.zuora_rate_plan rp - on rp.subscr_id = sub.subscr_id - inner join ac_yevgeniy.zuora_rate_plan_charge rpc - on rpc.rate_plan_id = rp.rate_plan_id -), +with -- genereate calendar dates, starting with the first subscription date dates as @@ -37,7 +7,7 @@ dates as from ( select (first_subscr + row_number() over (order by true))::date as date_day - from rate_plan_charges + from ac_yevgeniy.rate_plan_charges ) where date_day <= current_date ), @@ -54,7 +24,7 @@ charges_up_to_each_date as max(date_day) over (partition by subscr_name, dateadd(month,1,date_month)) as max_subscr_trunc_date, max(subscr_version) over (partition by subscr_name, date_month) as max_subscr_version_within_date from dates a - left join rate_plan_charges b + left join ac_yevgeniy.rate_plan_charges b on 1=1 and rpc_start <= date_day and rpc_last_segment = 'TRUE' diff --git a/dbt_project.yml b/dbt_project.yml new file mode 100644 index 0000000..8fa36d1 --- /dev/null +++ b/dbt_project.yml @@ -0,0 +1,7 @@ +# Test configuration +test-paths: ["test"] + +# Compile configuration +source-paths: ["models"] # paths with source code to compile +target-path: "target" # path for compiled code +clean-targets: ["target"] # directories removed by the clean task diff --git a/models/zuora/model.sql b/models/zuora/model.sql index f3d941a..00ad698 100644 --- a/models/zuora/model.sql +++ b/models/zuora/model.sql @@ -1,4 +1,4 @@ -create or replace view {schema}.account as +create or replace view {{env.schema}}.account as ( select id as account_id, @@ -8,7 +8,7 @@ create or replace view {schema}.account as ); -create or replace view {schema}.subscription as +create or replace view {{env.schema}}.subscription as ( select id as subscr_id, @@ -24,7 +24,7 @@ create or replace view {schema}.subscription as from zuora.zuora_subscription ); -create or replace view {schema}.rate_plan as +create or replace view {{env.schema}}.rate_plan as ( select id as rate_plan_id, @@ -34,7 +34,7 @@ create or replace view {schema}.rate_plan as ); -create or replace view {schema}.rate_plan_charge as +create or replace view {{env.schema}}.rate_plan_charge as ( select rateplanid as rate_plan_id, @@ -47,7 +47,7 @@ create or replace view {schema}.rate_plan_charge as ); -create or replace view {schema}.amendment as +create or replace view {{env.schema}}.amendment as ( select id as amend_id, @@ -59,6 +59,37 @@ create or replace view {schema}.amendment as +create or replace view {{env.schema}}.rate_plan_charges as +( + -- get all subscriptions with possible ammendments for all accounts + with subscr_w_amendments as + ( + select + account_number, acc.account_id, sub.subscr_id, + subscr_name, subscr_status, subscr_term_type, + subscr_start, subscr_end, subscr_version, amend_id, amend_start + from ac_yevgeniy.zuora_account acc + inner join ac_yevgeniy.zuora_subscription sub + on acc.account_id = sub.account_id + -- add ammendments + left outer join ac_yevgeniy.zuora_amendment amend + on sub.subscr_id = amend.subscr_id + ) + + select + account_number, account_id, sub.subscr_id, + subscr_name, subscr_status, subscr_term_type, + subscr_start, subscr_end, subscr_version, amend_id, amend_start, + rpc_start, rpc_end, rpc_last_segment, + min(subscr_start) over() as first_subscr, + "@mrr" as mrr + from subscr_w_amendments sub + inner join ac_yevgeniy.zuora_rate_plan rp + on rp.subscr_id = sub.subscr_id + inner join ac_yevgeniy.zuora_rate_plan_charge rpc + on rpc.rate_plan_id = rp.rate_plan_id +); + From 2901e7ff35abea7045e1286fef64ab11a96a1072 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Wed, 16 Mar 2016 18:21:10 -0400 Subject: [PATCH 20/88] updates for all files to use dbt syntax --- .gitignore | 1 + analysis/email/analysis.sql | 6 +++--- analysis/event_stream/funnel.sql | 2 +- analysis/event_stream/timeseries.sql | 2 +- dbt_project.yml | 12 ++++++++++++ {analysis => model}/email/model.sql | 4 ++-- {models => model}/pardot/model.sql | 16 ++++++++-------- {models => model}/pardot/tests.sql | 4 ++-- {models => model}/segment/model.sql | 4 ++-- {models => model}/snowplow/model.sql | 4 ++-- {models => model}/trello/model.sql | 2 +- {models => model}/trello/test.sql | 6 +++--- 12 files changed, 38 insertions(+), 25 deletions(-) create mode 100644 dbt_project.yml rename {analysis => model}/email/model.sql (89%) rename {models => model}/pardot/model.sql (90%) rename {models => model}/pardot/tests.sql (82%) rename {models => model}/segment/model.sql (56%) rename {models => model}/snowplow/model.sql (57%) rename {models => model}/trello/model.sql (91%) rename {models => model}/trello/test.sql (75%) diff --git a/.gitignore b/.gitignore index e859da0..ad2ef72 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ dbcredentials.txt env *.pyc config.json +/target diff --git a/analysis/email/analysis.sql b/analysis/email/analysis.sql index b50f0e0..08f3ef8 100644 --- a/analysis/email/analysis.sql +++ b/analysis/email/analysis.sql @@ -7,7 +7,7 @@ This analysis gets you a simple timeseries of sent > opened > clicked emails for with events as ( - select * from analyst_collective.emails_denormalized + select * from {{env.schema}}.emails_denormalized ) @@ -35,7 +35,7 @@ Do people stop engaging with emails the more emails they have been sent? with emails as ( - select * from analyst_collective.emails_denormalized + select * from {{env.schema}}.emails_denormalized ) @@ -63,7 +63,7 @@ This interface doesn't contain unsubscribe data. with emails as ( - select * from analyst_collective.emails_denormalized + select * from {{env.schema}}.emails_denormalized ) diff --git a/analysis/event_stream/funnel.sql b/analysis/event_stream/funnel.sql index 9591123..d2b8695 100644 --- a/analysis/event_stream/funnel.sql +++ b/analysis/event_stream/funnel.sql @@ -1,6 +1,6 @@ WITH source as ( - select * from analyst_collective.snowplow_events -- change this view for your analysis + select * from {{env.schema}}.snowplow_events -- change this view for your analysis ), step_1 as ( SELECT MIN("@timestamp") as "@timestamp", "@user_id" diff --git a/analysis/event_stream/timeseries.sql b/analysis/event_stream/timeseries.sql index 86e50fc..0a90ab3 100644 --- a/analysis/event_stream/timeseries.sql +++ b/analysis/event_stream/timeseries.sql @@ -2,7 +2,7 @@ SELECT date_trunc('day', "@timestamp"), -- use second, minute, hour, day, week, month, quarter, etc count(*) -from analyst_collective.snowplow_events +from {{env.schema}}.snowplow_events where "@timestamp" > getdate() - interval '1 week' --and "@event" = 'signup' -- filter fields here group by 1 order by 1 desc diff --git a/dbt_project.yml b/dbt_project.yml new file mode 100644 index 0000000..acc244a --- /dev/null +++ b/dbt_project.yml @@ -0,0 +1,12 @@ +# Compile configuration +source-paths: ["model"] # paths with source code to compile +target-path: "target" # path for compiled code +clean-targets: ["target"] # directories removed by the clean task + +# Run configuration +# output environments + +run-target: my_redshift + +# Test configuration +test-paths: ["test"] diff --git a/analysis/email/model.sql b/model/email/model.sql similarity index 89% rename from analysis/email/model.sql rename to model/email/model.sql index c3e260b..0365b95 100644 --- a/analysis/email/model.sql +++ b/model/email/model.sql @@ -1,8 +1,8 @@ -create or replace view {schema}.emails_denormalized as ( +create or replace view {{env.schema}}.emails_denormalized as ( with events as ( - select * from {schema}.emails + select * from {{env.schema}}.emails ), diff --git a/models/pardot/model.sql b/model/pardot/model.sql similarity index 90% rename from models/pardot/model.sql rename to model/pardot/model.sql index b3a0981..2f78768 100644 --- a/models/pardot/model.sql +++ b/model/pardot/model.sql @@ -1,4 +1,4 @@ -create or replace view {schema}.visitoractivity_types_meta as ( +create or replace view {{env.schema}}.pardot_visitoractivity_types_meta as ( --these literal values are pulled from pardot's api docs here: --http://developer.pardot.com/kb/object-field-references/#visitor-activity @@ -42,7 +42,7 @@ create or replace view {schema}.visitoractivity_types_meta as ( -create or replace view {schema}.visitoractivity_events_meta as ( +create or replace view {{env.schema}}.pardot_visitoractivity_events_meta as ( --even with the type decoding that Pardot specifically provides, actually what is going on in a given event --is somewhat ambiguous. this is an attempt to map type and type_name to a more event-based "event action" field @@ -83,7 +83,7 @@ create or replace view {schema}.visitoractivity_events_meta as ( ); -create or replace view {schema}.visitoractivity as ( +create or replace view {{env.schema}}.pardot_visitoractivity as ( --this table has a bunch of types that really should be event actions but are very poorly formulated. --the custom logic in this view is an attempt to fix that. --not all of the various type / type_name combinations have been accounted for yet; I still need to determine exactly what some of them mean. @@ -95,13 +95,13 @@ create or replace view {schema}.visitoractivity as ( va.* from olga_pardot.visitoractivity va - inner join {schema}.visitoractivity_events_meta e + inner join {{env.schema}}.pardot_visitoractivity_events_meta e on va."type" = e."type" and va.type_name = e.type_name - inner join {schema}.visitoractivity_types_meta t + inner join {{env.schema}}.pardot_visitoractivity_types_meta t on va."type" = t."type" ); -COMMENT ON VIEW {schema}.visitoractivity IS 'timeseries,funnel,cohort'; +COMMENT ON VIEW {{env.schema}}.pardot_visitoractivity IS 'timeseries,funnel,cohort'; /* @@ -110,10 +110,10 @@ It conforms to the basic email interface, not the extended email interface, beca data necessary to conform to the extended interface. */ -create or replace view {schema}.emails as ( +create or replace view {{env.schema}}.emails as ( select "@timestamp", "@event", "@user_id", email_id as "@email_id", details as "@subject" - from {schema}.visitoractivity + from {{env.schema}}.pardot_visitoractivity where "@event" in ('email sent', 'email opened', 'email click') ); diff --git a/models/pardot/tests.sql b/model/pardot/tests.sql similarity index 82% rename from models/pardot/tests.sql rename to model/pardot/tests.sql index 723a2b3..092c605 100644 --- a/models/pardot/tests.sql +++ b/model/pardot/tests.sql @@ -1,4 +1,4 @@ -create or replace view {schema}.model_tests +create or replace view {{env.schema}}.pardot_model_tests (name, description, result) as ( @@ -6,7 +6,7 @@ create or replace view {schema}.model_tests 'visitoractivity_fresher_than_one_day', 'Most recent visitoractivity entry is no more than one day old', max("@timestamp"::timestamp) > current_date - '1 day'::interval - from {schema}.visitoractivity + from {{env.schema}}.pardot_visitoractivity ); diff --git a/models/segment/model.sql b/model/segment/model.sql similarity index 56% rename from models/segment/model.sql rename to model/segment/model.sql index 0971507..7a10851 100644 --- a/models/segment/model.sql +++ b/model/segment/model.sql @@ -1,4 +1,4 @@ -create or replace view {schema}.track as ( +create or replace view {{env.schema}}.segment_track as ( select "timestamp"::timestamp as "@timestamp", "event" as "@event", @@ -9,4 +9,4 @@ create or replace view {schema}.track as ( segment.track ); -comment on view {schema}.track is 'timeseries,funnel,cohort'; \ No newline at end of file +comment on view {{env.schema}}.segment_track is 'timeseries,funnel,cohort'; diff --git a/models/snowplow/model.sql b/model/snowplow/model.sql similarity index 57% rename from models/snowplow/model.sql rename to model/snowplow/model.sql index 62a2f4a..e81a581 100644 --- a/models/snowplow/model.sql +++ b/model/snowplow/model.sql @@ -1,4 +1,4 @@ -create or replace view {schema}.events as ( +create or replace view {{env.schema}}.snowplow_events as ( select "collector_tstamp" as "@timestamp", "event_name" as "@event", @@ -8,4 +8,4 @@ create or replace view {schema}.events as ( atomic.events ); -comment on view {schema}.events is 'timeseries,funnel,cohort'; \ No newline at end of file +comment on view {{env.schema}}.snowplow_events is 'timeseries,funnel,cohort'; diff --git a/models/trello/model.sql b/model/trello/model.sql similarity index 91% rename from models/trello/model.sql rename to model/trello/model.sql index d7ac3b7..3eee29f 100644 --- a/models/trello/model.sql +++ b/model/trello/model.sql @@ -1,4 +1,4 @@ -create or replace view {schema}.card_location as ( +create or replace view {{env.schema}}.trello_card_location as ( select id, idmembercreator, diff --git a/models/trello/test.sql b/model/trello/test.sql similarity index 75% rename from models/trello/test.sql rename to model/trello/test.sql index 0b684fc..f2d5a9b 100644 --- a/models/trello/test.sql +++ b/model/trello/test.sql @@ -1,10 +1,10 @@ -create or replace view {schema}.model_tests +create or replace view {{env.schema}}.trello_model_tests (name, description, result) as ( with null_boards_or_lists as ( select id - from analyst_collective.trello_card_location + from {{env.schema}}.trello_card_location where data__board__id is null or data__list__id is null @@ -21,5 +21,5 @@ as ( 'fresher_than_one_day', 'Most recent entry is no more than one day old', max(date::timestamp) > current_date - '1 day'::interval - from analyst_collective.trello_card_location + from {{env.schema}}.trello_card_location ); From 559dc02e90e80177c5133806ffacd6d876dd79d0 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Thu, 17 Mar 2016 09:28:36 -0400 Subject: [PATCH 21/88] remove old shit --- .gitignore | 5 -- logical-model-flow.pdf | Bin 73906 -> 0 bytes scripts/analyst_collective/__init__.py | 4 -- scripts/analyst_collective/credentials.py | 14 ---- scripts/analyst_collective/runner.py | 83 ---------------------- scripts/main.py | 27 ------- scripts/requirements.txt | 4 -- 7 files changed, 137 deletions(-) delete mode 100644 logical-model-flow.pdf delete mode 100644 scripts/analyst_collective/__init__.py delete mode 100644 scripts/analyst_collective/credentials.py delete mode 100644 scripts/analyst_collective/runner.py delete mode 100644 scripts/main.py delete mode 100644 scripts/requirements.txt diff --git a/.gitignore b/.gitignore index ad2ef72..ea8c4bf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1 @@ - -dbcredentials.txt -env -*.pyc -config.json /target diff --git a/logical-model-flow.pdf b/logical-model-flow.pdf deleted file mode 100644 index f30c171221b623fa4d40ded63a9afd16349e634d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 73906 zcmZU4by!=$w=JcmSb^eBDNb<>4lV9d+_gBtU5YytcXu!Dq_`D#*Whl!^U~kF_r3dl z?~jwrp4oftwf5vBXU>^WD~U-ku`zR^u#vNp1C6av1O!l6R4tut%*g3pq4X#$qK;-p z&Opciwv-%!rYf;ke+K;W%Ii>;E`g0`0_%oMG`|eC(`j+^p>ETe~cZ??419bGK(nC&KYJp zkpuoCfknm43FzWz0@K9%UmaiV?0~RgHrD?#<;_ejjlKch$#wp9008XdJiP3B|4aJ6 zu>YAFQJ@Xb@w>f|i5dC7nNkw{ZcqM?y9!Jytoi>)|Hn4}pAJb!BaeTGEdRG?{|T4; zZbbgCu>UFa|5XkH{^u$I)A7&iUt3er*vic0f6V_^w*P4O$4!U)|5oXr?Ej%FXJ%*a zY(dWRUkoKIZD6G(XOXah&6=2*3DDH+AG(t>>@#MI;+}cZIaUppi>GRuC>d&mGfTjv z3e<&s`-uln)HrF(in>)Q`RP3X@hCf3-++??8tLTGW~HRYWSMJec$AFGp}Mqm7(l&?D-%M%OdpP*OQ2ENZTv3_k( zzvq3>^MzmS1@!g!81#IP7=0@`l=FJ&*DCn{@@>k2UcYR-UQa+TKAl7FH*ApvE}jMn zPG2WjUTJ;7$Aef<KRt5>Ms zY2*v&@ce1#RS)l;zme-mGeCPsnUwDddK4-rTikm|P9qfM{Oor}&xCZ?mInF2NFOO}i)~sE( zCkmRK`nT1eAGu;`W3gp@pefeJKA`oBrst!LY(&q7V*%o+9lm7`Ou^WKEv2=CE{Fs4 zp>0spuS#~MePDgw<@A(x^2s2o2wM+QzvicK@L;XGGN(>lu=!$@KKN+O+hmL1*0i(C zzt47eOTp)>x6o>XIH>7sYreAl#kIMNI%;I zq_IADaIezmmPye$>Tm;I4UchX9R1-I^YqrL$eD6@)Q!jo%7t4 zeh|bhY)(|LxlnhA|5O96JNG>o=I2k9{M4f4AG4|qsgwi(PNIRJ3$>VmlIEShkT(%% z_CWL8=L1CLmGkAK1)ufvDx~M3ET77KPa83Z_4B}<&bF#VNf8ef8;$HiR}kUl@#fb& z=S8=wn{jAcrQRp8r;C=2+i8>w*Drrw%-xiK`Ykc9pkw>yM0K7YGkfkom-L|NC@boIHK2dPc&7oyd;IkRBS+154t{ua=udl}I6q2O40rH+uqdR|}&!a^Xfi|Z-2>GJ8QL+3PF)VvukkZHBBoL>0sb71; zz%4Lwy4}!IbH*!_@y_h*IvmX7r|`75uFDX+blmG3?@rpf1$3xWp?aa|Iy`&V6&1Km z?39>D6)~~WIO7gfR?>T*MiAM&EHuViXk&^um+g|uUuoP@%yHbT0{C1!!CmC0Hs}G> zg0Xo3XC^~?*1+SvQNN2Hzy1~*&RrQiYKr(ItfO|zYbx8m9XS*d&e9?*uqBL8ga9O@#iM2hb_^4TYfSogZ!jMelm+K%4Foi`hIPCG*?JG%?ocUp(F);RNX znfcAgirletYu#ttsk0m%*#w*2_Jdy8nQeec;B{tQl;Y@D2j}6%fKmls({t_U~@xE!@E_F z4uNx^lm}Sm_0>aU9R41^U-A=;7!XI}+G;1Q*tNiVbqvvU_mK7?BJJwKBE1kWNgB9v z8zB}N-aqCdg0G)iF<>tU5lRtlhria87ueW$Tm;v)=9_*~jX&|_6;=~SSv2`(n=kTk zprU+g|E%xj_4w+fbvGk%wlY$2k*(6JRS&FP3z%2Tl_q@VwU2seNaMn}MD=?{JDu*~ zlGyy5>%Z*3)mYsbwd7ErhL=5UI)ROGyQVPg16H1v*>EL3yKj8zYU^*uiCsXfoj}do zm|?d@<;yL;)oFo7>d5a}y}lzdsC%MDPh37Kt9NZJnrtLV)Pp2Gb+S({IQJeIZdll) ziCo44nstgx&h^9vMJY4vrbcNy&TkSJa@m-z*xF1QV|6yZf83ZkO+*CJK|;_sg4%!e z48?DD#EzNv2oyFuv3Xe7l1ZXQ)6MV9rvP-C_D~S!O{B+ccB&@W?9{9pPaTd86Zar9 z))A#T?Y(LQNFYF%OMm2_FFB{|+Bs`PMdCx7 z+Q|)tKjFjye+Y0L2VC9 zPXtR2g5TnPG>^@?4h%D2K8tdz-<<;)H(gB#BFKC zRaQ~Yf=no9ySsF_;IgzAp?p&;i=FGhxFw}-%|)$F)4K?wuSPDd^o&Sen$Hn zmSOCss%m`PXW5qaFK5aJ-S^0x>rkN?{4qeT55LP3BeN3cU7Lv7Y}uiG@tPvb+}bX4 z3lk}BxN(#Y$;Enl_8Cvzq6?AM!u~bi6zF$1QX+gB3bZpoXPy36B=`7xNgVtD6^Dha zx<>nH5hjc|4hcRxG#fe@VK94-2vjrZgrWmi~NVWdbZ`C9bK}4Z&&N6Q&924bMks!szX(laxVbh5U!+~SEmnt7AJP^Sg}8K>CElazkwQ#fl<Y-K5Ezx~?Y)>}Lsd(^v_3bhx+8d*M6W+oI_zg?ej~f69 zu@+*yl}k+WKMP!A)w#=P6&}1tGZ2kIZvL?8%vpW=2?hbetGwgLR$3fap(JS`+ZBY; zLwzx@#om|Uxlp#Vu|}B(S=f0*&!UT2PA{Uz8$7mD)gZNjtwzL_P(k29@`bU&< zw$h>oNC}28o5<%s;ppg0&i9Zubqool!PdQ*A4ynT56%S|GCwcI74Dgki2DoXho#8- zaG$rHoAyd&l{yM!5iC}VDV)T26)Dcl0lfvYz0}PY2zX_D;YZIozI-DyviFW`{I+`ldNuRNJ+FLYpsDQkd9$M2_ z))D_1Pif&uS$JR2B->!Itj*lLM8};=Icw_7u~opw0H47mO&7zX*5k44#KN`1SU#lk zk!2Q+sQUcLk_QK!Xsd-+p6}r`L)Jay|fk*Ug)3h{7Vzz##<7x3qV1jm~Hj%r5RhyWQ z_Z~s%$E-j#5v9Eim0VuN$_c*`^D$tx!MQ7iLSB{l=13C(a=o?e)J}BIgYj5XYmF-d zchpT*&KG3}-?%CA^_GkAf>qE5mFoKbF6Ur+meHNjS1=n?IX z&o{^ux*6lJYw`|Xd}z)5+uyAh_aPbM7|!_Tkup0*`7cs)h5U$P<0%Fhf`wDj%eF&} zaay4qZ8`+P%30TH(|(>rudhnq57(35ErA8V0#xD#i2w%5sp65;hX(Nj>~_WKXyo8` zMV^x4fu?Rmrn%3@ejaJT<9*pz(vdA0mwgsTO!G23~L-%#F=!pRzc0N{U zLfD-j4uPCexm$c2J`yuHEdXXFix?lE$WoCC#bF%ck&uvXlXj-g9CCzJK8x?$YjQco z4Su&E`tutNZx+D#SI{9IBj|Dz@_=-;{bHt$S!SBKGDHdg46xEjs%jLE!GdCC?-kLf zJjY0k_Ohsa-&@2VMpL0Hlm_`)#Qq7`Uty%}gUjJw_g}@jBb6lDYSz4}&u_+5hC^@E zrH!TiiZ&hc)(ZVNKV(^(B_HE0X!o!S6g3!@YuIYDPTN?KcJgOIaFqwP+8A8U4_1QX zcc#xDFXvE%M)9h|p0uBd3foq{ng7!e%qg*%fjGihXNjIM*v^BYkpR@16Ci57 z`9i30iFzNhqo>jBsi}E=Y9~}4gRA)ImPS}XCzjTuckvxn41r1qZA%YLxZD&&Gu)(o zkd1=TF1@`I<3Vw{(LwY@b!6Xf5(V4c^wbjKk{kB2k8}ef_oJ%-uw@I%sgHfEQsP_N zGlJ#J)*ls}{q4^BC(qnlSReEPORe*3$!ai;sp9$azlbW7_fHH0xARJvzp@x4#82bL zCcp`DnSTBHYi$*9d*@8>F4BO&RvsR8O0W2%ee9nbrTw3Lha{y@#<@lcOuh&fQGYZv z=0KcH=karM`uWTS0W6gdY1T;DZ>(yFyz&ICc=ExXl9AGwxGC7B8I%52S%2rfy*zbn zOFqy4{e*Z_TT8s=&TR8xpJ3y@`v58DE)_+0eBe2AsxV^C1?Aa#pxukU>n0;v{Svdp z(+gPK@}cX^4Cr0!S|)kUh2;;kOvU@7z1S_LE?NA3cnhAIh0Ps-><=*?>!#4goIpC>4NJE7{x=5i*UVzCCQ-|(5BWieCMx9gxm{L-$zX@?1S`x+GLBa)f&ag8dK%qs(z;L zjRG`JVw!&%I|>_BnPQE+X5?Wg#g;7(x}$Fgms%*qfp7DUSUuDm1ypVZ+ZB+kG^RcY z4wn^0wpw{0>9#ARYLoRelU2(wVU)_gl!J|`qWy#|J?@R|#tM@WXQ$gEl0yx@s4W4Q zV!gy#5yeMEk=7a~XRkiC*x{j+Ym;s4IhvoQ9VgBkTX>`ev08f=e%G#Su<($ga_9uy za{u!4rd4Iaa7I{Qy|TpO>CrmYCJX2k-@stPU^FM}P#YD^ohMBz1FJPydEhECVNmLX zNK9SvU?_{xGR5wGoGJZ2`nG7q&FjdvP;iO2Mw@JbGwb=)c{GEmu|v|M;6n;UV@2}d znib@byNG;v38peL;6oXV`BmXDei_(56}&YwyCUT!pa+y?!sx37E*r>ho6KE_SYQ9t zpSx<2+}6__4J4UPEQg6LT}nuXv?#=bn-4d}OtgVlC>A2x^G;?>p9rHQG)? zpE7*0A+y4nPNP_daZlvPw1=}P4P9@ucuQ|Bsl38t!{W-EusYK{r2l+2SBSZkB-}~- zyG*R{DA8F3Cf-IcfkdGMKQok&K zg&MJjRaba>_2PcFiZ;sxej5*uEH8S-Qf5~>XRDF5D9pOuepq^?}dtbM3Cp-}M1ESzKXgc+P2)f`JXd z`|V*)$k57Jdo5e7{Pm`c%#V=|^)FnB27{%@b4XoVKY;72odLJ@cAEKo44;ceXRNFB zokGX26=uoSID!b%CxKhcr=cZZe(`6Mi8#^zVyO(n2fBmbhR?N|A}#W;?4s(_5rU$p z^U2$ibGL#*N!+PVcV|*o*pY6(9Ao<>e*|=X7!ZhDg!{IQWO8{(Da{`m<^!|{n%lVO zzor^DtWmB>e(7J#@7J_Dbf_A46q;FZC94JbfQBqM7IP(Tiqk)h{_JjESpoO`3l@Cg;q91MhE;U3IW^)Ss;noFRe&2uda4 z>C@dFB*roc0p}1QnxeOTf{YN(^~s0mLP?hGwAfXXs={>YK_kz{zh1Pn9JR*y&2XdI z_|Z-K0>4V}$$S*~PSUzUZ!N~*i-w};2F^GGlaGYJ2xTpqpI$g9T3h@AbylfFYSNIT zkfQH{itbFrPbR)CZty3*Ip1qyAU*08+a&tp27}gFojdn>-h@hk;hYr*NU~IUJrCpR6(oqry(Qd zn7EiJW21mdB99dk#-(-Z7GpMtPtk<8iK{a`3_cx?y1!NocTtri&27#z^=Xnii%M0-gB4V?{xPxQrDfurDLna z1yBtr?+%&<&_Lp9#6Hw(Z&t88zFnt`M-2{==614CFd$hau1YTwmmsF2zH;qQyRJ4d zP-^B}96_pfyn7_2?Q<@NKJ!%EnJftT)?4X}wLSst#NaU76VXB9=-zqvZ=!X|K0<%u zXH^N10??F34_l`T;1d_R29UA?PY3H?|htwWMF#-aw#N}n4>W~fs?L>tgtx;`da@Xk3 zi1?olEL{@5IDt+D>u!!`1hWFVR}qQ6h#H+W`^X|N!JIlBMcE;=4XhB_4p?BQ7S3Ov z*L+0Tz*Y>-0L9=|jMdRyDyJOaj%tPB+1o0+SNn&v(vUiDlQR>&=)mtVfXK+924deD z68_XDTLUMz_fDTIl=TJJI`e~Dp&==39ftDkkQ@lM{Jal(q8KT=@78g2>jZFzxd?_w zTCIY=Cj|OU?L_UAQ#!$#`UJFKg>2Z@R0bR1uz_HN?4GV*?rZk8z`s7;8yv_w)s^Rh zbL-{H=S-ID;KQK;mcKr^aS1A^Pmd_0#7>_KPOdH#SRvt_?l3v$_1H-RH0CHncN^B2WjK&>=L=bt-@RX4x>3N!3Dsc&}8JzSTicgz2*i?xo~{pM_un+eVrP1NO=3T zuV-5gj80A|KNpJAr~CPVf9;|>TBlD3*;wSq<*G1t4;dQvH*23`wPDz+VN$6@#&kv10CGzbLb%*{7yr&F=Xw%>Vo#{#7!~ucPggqa$-COPk!WCk z&#GTnOQAsYkeNb_V08nBU!&(WLk!M^XdK(735IX-@u8Q+!1lm({F~(}bwum=50SK| z8sJXS8ycw1;FYyy_kq3Na%~}Pt>;DzS53mzjnC9I&od4{KmVl#H)b@ROUZcZn3tsL zs)MCY^t{BYF5&!z2{0LSmUhg)1))uG(xhC;BUaddOOzQg7EG>VDVjH&`L~B$Kc$4D7k+2x62lDN>(Yr zEfF#RtOIgST7+BB+t2i>(*xdu8imHKNI#$1ai-=flbe2ns64w4j1sV2?SXj4Os;=7 z4TSMnxHY47g;Dd)YTngS0cs_>AdJ51*#pwb{S38B;O5@4tHtp>ea@=dsky$#xbA@@ zfa3IC^Kj7oZRtHzqVDf3We)5!Sq6!=P#p7TjdzYKER;OV#R~Qlyn$O$au}6^c&?y4 z%^B)zs*xG9B1J_*kM&Qgix}_v{tnbkJ(?)ySVhlkWM;Mm6^#CA4YGxh|u|em{ zFy}D~8Q4fep*hVcDlnmTjANm!uQdzIRo6Arcg0sN0E^(OHsv?3e3qaLP^UZ|Ov8-7 z;*4O>e8&#scwxd`-w|WNz8wvADQIsWjR&huja-$x*LB1!jwFFy#FRXLco@wyabm!v z&{Z~hjOSiAo!$QSBGYU zcD&3C`vU#wAKNq^-a$fgKJL>*#XZBu|61^&|5)Ae01#oe!MrI!o?se7-)dSj97oeE zP>(!8sLlch;7!H8UCM|&f%04<47|_gZgC)**{ek}A%oSEC}milpuZ?#H7G*e?py!A zE+*Wq)pvhIsds3+>I|f#i|`!kQ~M^DaO(}Ew&)jCGDguz_`!g z*EwQK(9g%M2084#Xtp9RdZ`1gw`t%YJcf?gcAPEWhvonHYE$~rwIaJqf=pmpB2ek| z;!P&PlwAysgPQiq>IAOCoUO8rE zur`fIE7-xihN+3NtOj`+PoK3;A;qZL!K>Gp3L*gBZ2Xbsrv?~(5dNK%!79Pm#Q^Z< zA0)|usWiXpiBzSt0RIQ9<%#0)943>}Hf)z>6Q?pcC2{rL84WGN`xQg64?p&3Hi_RY zu;mQc=3AKsXRvOnoZEWs(|j-;waKnHoEn)D3{Y!DE)d?27g{%HTdEFwQCtnWp3h_| zRBJ)rSJ6uymv5Px?E3=P(q-RgZABJ|MKb3D7)c~#T!dserIohsC1i+jE^xIV|9Q!z z0c`ypu?x1vq{Dx0w&M2es3zgm6V}$xl5C&+y-*Q9w*2CyBg#aPTXaW zJuZb@xf=8!OAFX?K=gj`tL9s4lzbKx6$gcT1tGl=uUK#;joa*MF8vddkFb|5rwT?s zX_r^TSP0~*!xbX<`eFD_9*W@ol-R}8abuRUoRdL4?B-I6b`8_z>7&|Fqgl83rh3gO ziXX93&4Oz*`SF=-*+K~XM~RU)cDKt)VSQ3#0HV1N2_-LoiB4`Dv07Od1DQ_lL^oC! z#Fhn7zI=+gE-{#GI9yw=Pdp-&%KMY+@{5>P-$%^8dQV$iJ=xACLKc$~34JlR#iK^S zGunCUUX*kx_yX#qMm?#&DFr{}@GBOQ%`~N!es?;B^uqs5uUx>({=TVPa~efxXFhM8 zF{x5x+f%TIt?dy`gzOnQXWdJ7cx3YKoA|7?s5EBep)KW6qY;20_(q}=o)~3abC*u} z^u61~TuH1H?uLYVz(^1O3f=AU1+E;^8?Aopv{U6k2h{!d%4+D_U&LXe-q(d$Y=2=k zU82**oU?9kveK;Gj%RF)CMn1@uUbgvd3&H-Q?Uw#V>^uFfCfJZl@?CE7dS;U)51jiry-`c(Byq4gYe~Zki-E?JIEPwKM$V@LU5@}) z0sGKNm2L~wS^F<*?^7m{Z~X1`2V+q9(yHG2E@l{sEeB z?|BHMt`xR)h5ubCpON@SHlu0#pgkJ%5oqh~G6(u{dva+=QuYe^u8@?k@9pBQih_$07Fd*6tYpCb#|2L2Whn6LtKMOSXM7VIW^;fk zCyyP^SLVBv2hQPKjaTONN#ooWQ6@?^y^go`UmJ@RUE;&F3&(o7lKKj6CUxUBtJybWEV=MlRiNHBvW4?}ycg zO?{USqg7w3tgW4ul2>mC2rC?Lu%+49)!Es4VFmP*;E>?nInx*z!W)Lim^=fhiXPH? zQ6~S=+oe*g{5FBrckdHUpyk?F7(nPiT$xR{{iI@SYE6@TGIblVF-ie@e&%AF)NxA=E%sErje3?tdFxN>usVt%GA0xi=s%u}70Daqz^q^2fm!k8(#NkuBD(mS8FBN0iNg zphR+XVkx$ElnojlD_0Lp#Qa%I+tr8N@TGW#vGMxk6}Kb;LIyl~O1E~IsHn~|@%Z&` zKO1dK2NHJ2EHi+p*==EQ4*RKg1m-DfY0u+~DtWoq;c;W`4H>rGSq0x$h*FzN`4Q@Q zwlRL0CEtip@|mV-D6Q4|B2{<l7GU5ktc|FjbDSiRKgw$KdRDStRzzELv^l1TgRbsR(IQSl?$-n+c*vPTwv1Ltbb(V7K z955}*B@kn;P4sFO?VXXcF_sY*6Q(%I*QU8)DJCeGZDszg5i>k+me74u$$v=tG1cX6 z{7fiT#GqVamn`J!hU`-8Ilrho4HZ5N;K70ELMTm?Nr| zv|(Qm>|}Z^dNS4dlSwC%jP&p$Qw7tudh@$V;@b$Rw9i8{(;7UDbt&p^Jyu+;J&3qA z4(az^O>~v3oXgBL4gE8zeDJOPhabwsa~|C?id)-4YtJzFqK(P=v+!HA(L=2nv4)r6 z9;56vbdn~+eQW`P>6fIU5Dp0_|fuSF`Ki_eg57u^Jnj-o-uMB(s-z1m&64Y;X7Y zl=)`WGG8$B-B*SlvFzcorH8s70Qp0h8o!!AMF?(l9J zmf5h+vHr@au=nQdTBiRk>J32yW`%oX z^KJWPWg~^>BE_%2CASi%x5OSgRzI)@R*L=VBtMvXdfwi%NH)!WSM9AMCUl#|)RS01 z^0pZ+KgUbrTy8?(Ea>q$-Wq3ox+GSD?0m9G-G3+KYh2O`OiMtXTRXT|AquKsa_?eH zH8PHxlF?{fRAo!#FX=6y3Z7!E{rq<=(h>&O zG3}D3&z7t*QSq5)MBG9Rq9GpdNPS1ylZM1N_yh>3lgUL)01|h_bqVE-W=pN*R+o`l zPicbOKSzT9Y|gR}k&hU%tRPj0nJGFHz7po6Liy~5AR8ApxYqf@AU4c%W6>%0Ej z?4UuZs#4G!h-em~E^*Q@&n)I|F!fF}b6-mw}xa*xN&Z0tQayk`}O&%0kV%*r(KDd%e_i`nlVd4l;XcoU#THOUWVEt?od z>0>IzzZE8e{#BT*KtE~@avR0fi51HT#lnv&Liwu&d=x$dV64k*ow;O$N_rttCExA{NxRG$ z5Ql_-y4ix4j)MB1=B4X8Bnw?o@a?3%b9#btYqolr!Ygzphy3v%jcS>;D`X=_uBKkJ zoBF<3AbMATk~!XNRy??^*eBTb8ci}d&aj+>?3e@a zdAD%Q1jWj}uEobY{Ah2-nd$4n)z4Jj%NA|0#+Vxm!(fzc7|f2oQ(*Kkb+h()jH(Mz zg%(P2u)98-o1wa%pvX{m%ATPB9N99TAF(}g_uaZ1?H8r-d|uuc-aLi0_j36_6JM@A zyuGE8@9=MS+kTk7C10Y3yto!pGKxB{Wc$Ao6)(pZ0falxwpg9TJy-I(IrLZ^Y6{R} zJmuma3W@lCDd<1|H>UFu>fq+hCi~7wh4r7!aI999Eo6&UX8BAgJhYNYN%#V@b9M8R zWQ@Hd$_Qs%VS6Q=A{vgWTH)ItvaQSHoK=08j<7z|%&GB|W4FzYh=ps!e6d%2QH=TG z$F>xwoOU49HKW3SO&TYXOrkb7_X|fhru<7p*oiW4d1bhfrAJc1$Rj@4vR2*IAfDNW zc$g)`>slt4cxX7RjBsQ&lTg5yXI&-7xn=eNi1>?SZvUllL`-r7RH078mp}E*QT4%A z#Y)U9KW==#UY#Z)VkWpiW|lPAT%9I;QYFozr=Sg)AikYyZ3q@cwV@GHUS6YVZ*FRq zPdU9PLtrc^Th`w=4Rege!@fOaa_LgG&ORA->a=1*GslJLpOeo_Peg9&v{>~foVuT;*N2F+%d071~&EWeLI(xpcSCZ%&!X{!{g(UB~v0Hs5x;?Hhr z9bACr;4q)rn|BpJb>4DD|6h1!9*@7Vzw?yS#TkF@3omGs^e%T)g)m-d!@8<|VdbjQ zd~pYt)-gt6Z6_3rJP;T#L{NKTfT^zy{nZ& zMmTw{m1ZGknKGhG-6e4^tZ*2HCq1l8-6nx2RVK%oFCyZoN-wCl+*B#&?C$KcU0VK{ zl<-`~K1J4~UZ>zuqGQ+rb6}EVUMMM8mfI@5y3tuIb5!+*?cza|Y@7-NRztZMu;nev z^6G#AC6iq;Fkst$jjV>vZ=|f6L9)5;aIL_lYz#Qu zz>#s{8Eba(VVvbFY25iUmMXI6b_&TUY@mVGcX%r2fm+fJ*6|WDn4U**PK7cn|SP{oGf62-f=l~UCVyaexSxx^hK6j|7gDUvG6{^z@8pq zIc{L#3V&hYQ`ryHgO9MGd05Z*H8yBhGy9(UIb!3Unj`%=p3v>T6Z+)C#(%tI8K@}8 zZKaSLbAWa&V45=z|5wtUtm|k4M;pcG_y~31;V6DkQ2RMCXm=5o;-7qbfX`0Krnlgw z>;)Q3k9&sqP728=2PkN+rR!4G?#ADs&S2s>9`?}aJjc9!`K_$b1<-u@b;<9DLocc` z!n*SH1%Lc`Le2&Zo%4Idf4TC^2uG~5Kb!Di6o%gBw08JyHS7;FUGIAD>VR8r7Vx1& zBQzwpHNA<+!Y@qf+co!Mi*keak${kf8z+d?Q;h+BWA3W0 zFX@A@r!Nk#sge=1K^xZ~ByTcl7Pa^H?`QoULfatpOQ^B0cvUJf($Hq)sDeq25zuo` z&JVl>y2YW1G^P)zzHQJu*)ffmYrmerfP9I}`Jay*Lecf@M7M;o9$nGbZ-m?N&!Hh+ zX=8XYq?3UIzm*X3f1-1zxUgq+==b63(vC?ppv@KpGNCBrs=i}6hCJUnYs`Gi>o5wB z@1#|IOdt3?;L(%Ea$(7kNX=>C+2t!1n5}oQHmsseJL4E8sk4q3z2&dN&bpE#^Ta#O z7!l5$suKHC8~3xaIg|T40japwz;ULPOgv7{yRN+#ZXZa=%L%6{1F_S}bG5|_P2)+v zNpuTu1&YJRQUuXCcDt#+3)_faX&o;+P+rAXsmLU2hu(SqoH?LaKD1Qnl`gMK&v|_8 z7(ynh^SYEsv0D0VneQ&;t$0n}Fj`d+5OHSVpLl3_zh^<|v^%E$Y?(D{Z0B%IWIm%h zy2t7KAyw-*RouB~=Y>Mhi6WV@Odijo}we3nq zDKf~V(cq`M6McaICZ4F7ncl#+BumdpKNm`B ziycwmT(nPjcvIeq1QD?t6qJU{0=FUvP!XB>wd>PHkY^Rvx2k3HF~Q94BcxnnoD+f$ zmhVKE{FSC1uLoQQh_Kr#*_!6IKA-7U&_&RxMp1LhI-uFoo0Tomg{;>c99#PE&)q}w7L6`8HR0(ZE|^EE-|&|v6h~8huo2!#*-$Z{ zco@m+d9gSl?x>x`%^fwa`vHkkjmqjDUsAcT7n~M?CvJ5szfXS0WX(C}P$uSjPv~10 zAD(l+Tr8>UmkCezWf-{$?J?K<>(~ADDJ?pPut!fZ<-!w4lHt}9T41$mTt7}s`Yg+P zxcB52$@zMCI2Wwl39T&%8EajK?%rqHmEFI@>UjhgpF)J)d(8jxp8I(dvN`NOC ztDh*Vtz-S)0?ao2$86QM+K>CU&?osTmkYicTBe=nkz#i&mF_X}FKPmNWf3uNo03Bz z_+LuhebgxoJ&Dxfq^G-Ch^~n`TCHbdy0zi}dfsi(mqx=xn56#IzE&CE32B^fPjlJe z4RpjX0r2?l!UgUH%O`# z?m5s}#b)bshGl=s;yG0=2(*;1^!ahg*OVBHpogkqG*B>S&55zk1lIeah*1jHs1&_E zJuT2!y^XZVq$dnoecp;FtTrok9Y17H%MRI-p~VH-1snY2rb&G5Oo{LA#C{t3-b zYw94t&*9GZg?gMTh35V24QsCY0D7r|-*0#bdZ+Q7gfpB~8j-GecrvoBmXmdl7tV69 z>p(_yDVPYZ@mrRNe2-O>%g~&+c!lWSqlx`VT@JOB^Ad- zvZy0_G5M{oaaFz6`C@kc?qsd9vYHrlD11g}t$((%RKqqtE#AvL7OOG*Yf;jBO6`G! zfv>8F8KA88?N=E*)E+st_mQR3Jh7D>va_>dvpF|~@FU1hz=FH_6PmG!EQ(SM1}ros z{J}~Q*&a0N1f)e+(yPtq{vBjvQM6pKZPh9^H7;<&3^QiL1t)!ppsxZHYRFKRY|I>pb z`hr7J7u@6frdSyRxb5lV7;DpRyVM_rT^IO8UT?V0*o$Ilx2Ii#unO`^o`fH4cZv)SoQo7km# zzeiJ8Ttaynq#_sI0x5SN74FKT-G*=0eX3J9Cd5Yu{Y(N^Ssco8ZB3=fP-e)5cd4d_ zN?%QB>95JKzkyWmOdU>LdSfSX@#7WK@#v^GW`<+;6gt4C0igs;NBF zI@(`Fz0$s{J}aKl!wl(y)%Y*?%8MK(-39+I?l@@|*jOd+rU?FQK-TBc-Yd~F^uSW2 zYATm1hkDzw5#o+7+gC(#mBII@f+=XD1}pr&OFkoL-d^IH$(7jZ?JEn}q#W#e(Tirg zTEjHu`$&eXGnBCT2c)tvX(V|KiHQ{Dxbbk%Ikt-a@EEUq#ntPwE$>2q<3i;{k5UIQ zkTO3&WAuY=vY`G>g3kMDPmDe_GpvLxGF|e!zkkfkEF%RBSI@RtQMP}h257l+ zk}$qaX{=-#4>ci|5jxg?DrN?{hhSLf$1(_n-5!oYQ2eGkewFH1x5GP)f6$$!cM~m_ ze1C@Zm5h|YY;)YAPT-5L7Mpv;*dv@xW7}&``%ut3^-itw{RkW*TxJA5cTG{k;I0yN znbqoir?ti|h{K;trc{%K#m@i%ZK z-II-IhKz*ryowR!TJY>3-u{S&qaRiQt8|Aa+D4bwNH4uswY`uZ_?i}E#V~)El!Ba0m?TR3cWy0}d& zW3Sv_XF)#SYi60N{oUF`P_R?9l3hDIE{W%%L--)66I5_m)FByA*ZKj7Ek$o#RFQo1 z&g6x$7nI+QAL9O!dO=xwMX;n6u{FGw8oJcRx!P_%>xek0HwFa5x@Pb zY{$csf`)vMtoea3l2AcFKVH8XRMNOap%lba@QJ>(NzNh(d?;1CKZIxKy-L7R0qJU6 zDX=g%M8^RAAI`oys?Dzn6D`GyyA+DMyA>-g4Gy8Wy9cMZ7I$}dx8m;Z?(Pno-}mj= zJ-g>y{UgbFXXZXLbCdJlo8+0f_>#p>O%1oWnAHKHD-FNbFFEQjN2rI4H*l-MKR3vK zT-zDzs`f1%<DBXm69yyh8cnRA25~ zE>W-2Xj5y!p}25gYvCkrv2g745Z*h)C#?Ns)yD@hTT>M30&#d`q1&ojIkjHo3g;6@ z#q^?B8U0Jh4Q{ZR6J62sSyo8cxxD}3np~?@H5cf>HlgZV4pgWtQg4GUX|ejAIyr=h_-H$a}VMX_`F{z`C3 z;yUEKU&2bID_m8n2gz6*ru#u!|@XVC(+9*Y~?rIV;;T;W3&c^+8ddTtI5 z|Hgl~MqW5|%6^)UIs2}=+!c&AxUV%-F_$C>)Y2%hSesEv1%vH$tslHQ#kYdhDIKX^ zkBkX6z&7ZK>;97=zNZQhT%A_W^gXN2NpgWcIGxYjs7J;J^8&)*o3=`+PHDB|w8f@T%luj`FpNdwX!n!TndV!QygaUV+}@c| zLZhU{y5QjTJ@v>+F1-zjp_}`^#i%uHo`3!i1_HzX8VC2kMELIp#76%D{!MQaRnzv5 zSma-x!QvOve|ZLr|ARm8fBjzjUx$E01iSj*B7wK&Bg$&u*HyKkju0duGub^FXi%Zo zdruRiD;Z)~1iWEFFq$HLGPo3I0aiD#zE zg;x^FG$cf$X>Kr%ace@Bkh`6hV`_%_G12`KBCUP#9J8A{FPW&Y`0OX~MT3?pFA)Pr zH$HzlD&mNah;AfD7(9Xbi!w74*oOd=%=viOp^Ff_pTtF1GwORuv-CMJlKnpO#ox}u z-)2(3AgH)u)7OM83D*zGStlJGQNQ$A2&2f?Sgu{b*~0)Y3X^JE;ZnOk0jAzsRDQ2> zWcLtJSGpM}5S(5)SXrCuxL@Xa~w|+GtTOIYvh>2t)=I>TGiAO4^?=hz1lT z!joa>CUg94@2A68wLZ7H-=@@x>NU%nvMk+h-z|Y~Sr&~_g}_ZE_Ra?W#?~X=(*<@# z)I3?0ySMc|H3_I}5fdMlFKiZMYo0_a<+`@G8BuuYl#Vvm1O99Y7*k1H-Hh?|BYGr4 z`KSL%w(e_In_8m??f$%eP`}G+pU9L)i!3L+}=Y%nj@*R8ygyIY| zg88V*PQpT=WQm1|tGrV5KGIaO6KAa}`kDB`49j{OiKmBfr z^f88W!EIF&2R=6aHj~INDMfwFgS@G)-Pzr4>2g_8^0&OHY$;BhCaV^Adc?=}{OS8M zo{%xLKogAk7}JR#ep=~r&d@bPnM#zH8II8T{ds?*$D0iKbv~mC&^~UJ^5JBGb6UyR zO=CE=TtA#5vnPswd-zssl|N2N3!!U#o?yyxX`cz6xLLFWl>x2cW6PTDralDs;@Tm% znfrMD7-ye1pz9aGlW5E#)niNRDguf0Ql9k8Lu;O zN{BQni&bpr&MuX?va^3J8EH|J4Kem-h*y~X^lb}JGp(OiB)x|jXX^gfd_`VPP$peY zwUo8i9JK796Rep&Mj@A1JHHfupVeJHzK@;^6mB2i-?oZmKPalzxGbGt(nb(rJGRvK zO5m}JCni_J?@}wS4Z*V0KZ-Yb2&Ml+7v^7GlLI?mp=^TT)cfif2j1`MjV~qB`C;ub z;5sLOXTva~mquX5oG*8ra*~7Ej-vBc7XJ~?iLhKkP^(rpA@y3f{a1(39FKFGFo=G1+Y5q(iYltx9A;_J)$;?s1#!iO>|qYP0G zHuT?Zrpv+cxbKd3`%3QPX?gl`a2ICYd7i&0eSE%7>#*wHmFrdcv!&rjtpO9E>N$Wh z=)ungm%c<#4q;pd)xJtEFs)ik0)XW|aK> z@ZxC1r>LxcV?~F1q>thOSO`KXTsE_ z2|ytmXRYK#0DkIHu5UkB!ZhJf0oKw|&Y6y&aXcR^BGFDUQWAhGRV4ISIUUN1ljatY zN*vCk`8KQZW|;-Z5irOb&F|xf{|Gq1nOlC^pv2p$2+ew?aHo7NtH$$ika14eXImxT zM;ci~vcO8k90wyl6)4A7pNf(OoVmMhL+rt+|Qw)Pq;-Yq0E;8}^( z>ukVtaQ1*w&mz(*l@AgR)Lkt|D-dwq(qEbSdwtAU=1d>IN2E|0uY6jaEa@hSfsH;O@WH_udk3(v z{N&yx*APCoiTRC7$be0wwh=sy=?gT`cvf9BP9|azV_9Ui6w-->)i@F@mGqpoy2W9$ zi?RtlmDR~_vU4m3Z1f{GrJ4q8Q77LtQ1aDa$0?PMeEIEV4@@ZHiKlc5W2EUcrSjR% z^QL!34^?BNqa0WRdgkoX) zvG`VECsJ1VD13jZ)HO!>WKfE>{H7>xn)l>e{5Ep=rC4@|z5xHUhE)E*<#^A`OcioO zRYu~G9Z<2U^fMj%Ev0tn+~Na#w5QGsdECmY4|ovSscg5y*`~k zi{|;8PVN@i87!;<*1#=f^S2j<34mc$jof0Sl{QxK=%i@H=MLX)U)38aDF4|SAGruN z07zp1ATjU2WI*T9+|Kt~gyna%HdX54VjUDxTLJhPi5}!q)lKDjl}`?MB0k!6jk_ zQ6(WsH|J_yi-oAxcb3-d!7@Ru%2bXMySwm}#;4rVgU|kmi?J@J!r$~IXgs#C)8k|E znfMxtnp^{F{ZBTP5^DkY?N?tckBadGa!r&3znBI;3E$l{oF`K~IVydYKQMv3GZ(uJ zaVm{uoe#%Go@j>Bq-IbJwf>IkatSWREX49!o@!H9V(iO>%B_2a$QX6CdFrxnZKdi3 zjiAQem?Z=-OOHgcphdUs4K(H0lXfI?TnRs6C`<^-jG$u5NcA=G6ZN0sF$j&K z!AjK=9ks(f}P$!Vy63#Qlk|U_2%VRE%BsB?*DMzk3JgmZW5x2@u zaF2LK+)QLu?wc?OU1HS^hBv+@O<6v%?5tlKq=BnTrmmp_x>v(G;uA=c>J#G@Z;;|J zYJFAx@XpV*1o2pilcN zO(#na39DFfQc2Tmw5c?W5KoB4%@iEGME@GDiP;mRM!|ISbzcy@mWQ(%`_(VJnI^c} zJ-A^GP^5<7*ovXrlV8?nNF6Em$$#E4LvddDv~_lg{kjO#C&*c%n7Qg3S8svF*GF%= ze9AnmWI9Ls`n4PUHQ|41qH+~|eJr$~U1%pGmW@zmi-j%fg5gpCSsX1%*IW?@{-ZR= zaO|uoEs6Gwhha~EetLIrO)9M-qYuZ2LPV!Ky@&+U-Ux&`=SF$omPxHvwm%`Cu!^?9 zsPmVF=v=g46x$hJ5=;V(V5%rSLxVHv@dbFT|6uHq-l{pwHnZiYc> ztu)~5g;mdtLA>ZUJofvw$&EfYMNgK-hLB88ma1og7cqFh3eF+Yub1czb-j#E?h*#z2KbApz{1y}axtRDk8GRO3BjM7QX%Dj*}eGY0VKo|Eh z6RMZB;MnRvv0pE7;$*dHffLrK=KV;RXmIzpY=O*qyIb*Od7S|h$z605OH2e;T-i0KgG(i5vZDB(;vlgW|iL=s|q-F|g;c@!id01{AdhY)T7D(AxZl`(`DLoEhKIUD$ zeh-DEgVVUqA<;Fq(H(u(D(ZJ^{`d4$83qJHQ_RswLqiwwX(>gwT)3R<>wM2U{b87u zM_G99?XA_wm5XCxl^l!G37V8fo=)*a&5Bp|tGb}Cf>yBqg?^yZ5{c|al-sVCwDwcb z*PR7YxvvC+<21(Yv=muZz^|~TDWprYA?RvGL5qKjk`2%Fin3Lr&-}&e=L`B}6)LTZ zt_t)lFmm>Dwok<}-K$R$u*A1pf2_f@FqK}wkYluG7Z_V04Se0ZHd|m!_(9XdANuy^ z+y|S%oknYHZC-PA{m=NmYURNpwY=-;mzfZ7dla{GZJ6Vz0z08$0-JA+*TjFUzo2zS zrVImun<=rcPD)!ld_ldHStBZBUTo)RLg8iI%<41?eN(0q{*uEh!H+CILs59w;6fj( zA>|eU=kg-=1*<(arJe=5bP7GOR;tQ}-J@`IvGQjd>BJ1m2J99VSFEdT-2|Xs8wnXo z!|gxZ*jIGqKeq?*X>ZgQ(68uff#fD4Hb2lYoDwZXjK;_O%;h5FRWc>6Q&!~(m&m-p zjAA|$oYUaN2G5`Xq}uTs>x`b*k^WzaO$6~S%wJ19_5PO zc4#d}+bkuqIHRovDfKYGW|VpcXhPqzrBP(JcdoU=H5RMLrt%hjknAYOC+>~eUJ*=XO}JoOSuOvB7P_c}6T#)NH_ zEEp)0e>gmRe6M4VIrcgd;Y(U$`irGra*lH=dMz4vjW}X5ww)s+g(FlW)EAYQD=gu4 z|NhUtOZNra+|cb=86{1u-3eB`oelmV8{NN>On*cE{1?J0O33|VFq^d;Oe>!TVmC)k>j-X_Qq>1WTq1uqNw7ss zfqF0%TGvBnBW)3&Cl=g+rQz#(Xx0u=+}eYLIUd%wo6d5G1=BaC_2lblB=(MKOL*s7 zDh+vVw2vfQS=5?5L@e51XP*_Xr4y<(lujcC#OSDisJbXxnDntB`g7 zt5EsP6*(#^jaZ<*LZ^$C6j2<8na2pe10dV;XM zy0McDSBt$qUH8&ccr&|3v2T)-2F__QP=m0%pxh&p`L+?fpd9jo_(18IK5w(4h;m*~ zE|IX&+oxomQ-{H^0O~`{M|igLPpmE^Fwwi&R~Sk0W7BJn!>~fWJYjSbI6H%1N632T zetvM1`J^0Ej6pdTx{OC^pNDe7~?R-}F$k7E3T;I&Udsl46 z{AZ$5;(Lp_pQo(1)1J%wkxezacOR9ejl*fj)qAMI^#geez1`D=yUW|hL%3u>Ak%yT zm&?0k8Y66iZDPy=QPPKzi`E0%dYI8Qw}A@p2LI8ojfd$YeD2@ofu%DXiah%MB?G4s znu+8&Q%5b^xsEMBKTD}qULB_BIoa75SG`{}%wK2ubeKfi$gRq848hw$ zr!qOLw2X5gC2EmN%eeiNaP7=pgqHJhhvXS{P78t$sPF`DOnw12(ZWn44fT?!NsXIN zlPQ|?5>F^}0>S1h2*&mT6wH+{2M5OAD5~CJc6M&*FOlklI!~`iGzAyWXTL&v%-DcA zA#?O@-YF5kkyQ#$28i$jYO#QQAWi#wo9(b!nu^*eFJ5n#jp$%%_rPf5Pd35BEKzu5 zPnf9^O}vAU91~v^7aS6Ptdrx1KU170d!l+}4_mFoBd!W4hfCRLE=-2{XzB|p^4T%X z+i^MIHSv)7ru4uUxyUgdRzE6D;=)FuKrkn4$x?s_9dFocG7p{LcugH`!?#r7F2f2H zF32`Gln_N@@r2i^vl`^@+mC0Wqu^&->&ZI4Hl`V?%|9zED0YM`i2)?;jI3v!-db!b zVmdyWGQUwMn5gY|vlMLDY$qV~&)B?6jVk$t7U|cnDkY{5ZGmZi4O(hvfoM`^Nnxj2 zSyMrJ-y1IzN1N&sHwP`DgdJHb8NqG2R=Ce_e&;m!&6nam7Jot9P)tBk+9-9^cn(7> zS^L#-eRNb)8&&mZ26AAjIL4=+`FGTXz9v(etJdJlpQ7jh#M@xod-MP{#CLK2bxJP@ z0b<-Ae_z`?U_ktbIdNsHvnA*UJ@1ISN@v}qzl~x{3;DL!cQ=msTaS)Lp!H+Qu6xSx zr-!dQ2KDK?*D4tvCB#lQWe)!+9NGdy`)vu$W_yUD**a2pBB;y7iN!6@>iRSJl%3RE z9A{)iOG%oX%1CKuhz%vBKD46`85bY|lP{`ip5+X|eNEMEs$?z8%A!FLhfH zhh2D|*VXMK+JRnvk}YYU^G@0x9@|&i!pU(l4`M3fA3Vywpc5-X7M=#gl5#7lz?m6S zIZ;Vi$%Y!Pk#o7kTZYZ+t6Y<9SQ|!fhSbuu#0$FI6Yw4_(iCanr|`|?w1UPQ)#y0X zo$v!Dxg_!3VaD7v8zTF%X28$bZ2@UTfl$w@0K2ZMcsov}YmK4l-xtA^ve{q9{8nYv zQW0CLvd~^JYRBxzK_nG{UU0{4lpfq8Kf(nt=6j@=clT-26gbq@ZLVHUGxF3M6k(p) zOJOwZ&F6oZ(HE(wt1>!)Kn@3RlB$0a_~nk2($XKqJ`qcsJzfpo0Zx5vI+mp6O+U>e zS|cv3uDufZK$S1EY|jq}KKYXn@4TZ&&0!*ALb5XU;77za%hb6onTha1?t$3xbQ}Nt z`SQ^r=$@Xfi*-mh9rkq}X9U!w$}Ax8x_E7cEAjr+7Nip7eg$R7?Oh=Jfo)B>>o@`%JAx1Twbtr?yBm^#qRrzz2!nI7^&nG+hMFpuYS@T*{$P(Wi?0EV;L z>LAwevz<$7$Zq6K0eUmIz9j2xkx!rN9zv|IL+zbI=cWz~AD~SjrrYL$`2{9vqFVWD zA*P0r&uP$eDgAs2g{?Q<`M{8Fr~;DYB3{@sXreM{ZnrZr)RLY}m*M+Dz497qv57dn zafn@cW(Lx$N)mm&=~|{GIuk*cLG<&_2Df@2v}AI{G)$50xiEA_WoqFz9cDa22WtV} z6%`GhpGy)7)og|(0W8Aj0@Vgj?r5Z4JZcxxo(@CYgD_TzG|RMqcym>kz+IBtq`xe= zw2?%RPJYjyEti^)>aY4sav_LzUndIgz2ad}QM_RnO^|iL@hU$G@VHmsWSKh4`6C&I zq<|kLa`P9c_?S8lQJRbrd8qfYkFcyP@|7m?(3N#d9Ab_6FBfK725V-f zRXxc^lYqkaKB<{lu}I>b=(zUl#@;~uP_SCCg zjp-mC2h_BTF!86oieGI6Ut_CgVr3YL(v0n)_6dhN9 zk#%Hkhr2uHX~K8);b+hHw=~2>im)&w3W32nTFZ@kP9fASt>nj{+e8A=QpyL+1DfIr zE-1M0%yteO^5i{421Mlc#7(eRT64HtoP80~;yS1DYIB|@ZO0jmt-`vy+(BXW{1J`; zeH_uaJiR+B{C@51W4z|^ayH!AMgj;x)yi;fRo(*@jUJ=s`usk0*I2wrOCw)__LNc6N8QB9nn~63Ocqr zOuQs327q)I33H_i2ht1CH?5;L+}DnA8gS_*pdLSYzNXD)kkM|C*c2Ql9ehRzfBx;=SMq?JirztHP$|sa4}3^&?#{nNUX(n-mrPc#D*4Tz_rD=QYvh(N^hu9jsed>~G=AweO*K zTRNSKuVfnwG>)w{46vJyi9V*MoYP7Krk21}OAzQro_l6cE3m!5?KardcK%qU;qI9c z*qoI!htJf+pzY4~1%cVkOvF$o`8r(ff?`@CY9XKoXH(<)qV9l)J2p!ZvDykPe5)*$&{73d2yVZ*bz~*L7`4_RKx6`P zQG2e9OL*<~0`6ubylryXoeT@pWHEo6%;89)raDeAv{Hw>I^|;qljX2mSIEtf4&{SV zkSFfm8)E!FJgsd-9$>eGIp2$8h&AcT+lQ|^n|qUK zTplaOlvl@n`bbwkGFwE3YzJ!L+X*DsGDsFkd5eynZ2QB&E4`>QU|O$iJIT}SOwv}b zy55lGj`?E_*Ei}i=b}8ul0GijO?^2#r@t^isM~jUy=QHXIDKY}!=N>s68=lkyXqww`?GuWifG5d#<3K$ACjxRR?WlTjFbQ(k}T0og9c@~b%CCJnZCGm z&RfVtd>=^61$T&fbdrI#aGu{6RxaclU8gyw8-KZ+>#dxY?zg3zf)EMeW#D^M*&=F~ zsWxkvQ4f1-g(X9RFArw8V@?Ysbc4*Hsw>_Kh_P6o3 zT7Fm9K*1-!M8o8Zv=kfKq9r@XrVDllNc4uS!Fmuc;tgBrpS@os;NaMcwK1O7xuZQd znsTav{5!VNHHlqv?oA)a2E*k~dd;GU$C-l%=^h4&cV!a;u*&7~jaRi2^@Q92R95u2 zJKWp5shq`)ZLH?zDhf{t!=>?JirsXoz2}Ky^$uj`x)~C^^1G6&tJ3ekqLBOT@;8tk zV)P!|2r-Mrq&*9=YG<&;XV)iZo$0*9{UONR>F=)9Y5z?dK37c*+T%NE0E}^WWLFO) z`~dZV%Fxi*02apy+TgYX74{K~0_RZ+qWB};Kli&jU6RL~46B}kWqI?W2-kHRHh&y9 zrTKvn50DJX4^kG)J>O472A}++ehZKIc0z{PjN3HrJ)+fT{Z&t6qz8yJDj`Utxqw#o z(l~Qk(Zm>;s5!DtoTd7uo1}8!qu5>npH7P@dpkXPnAG??x3EvB%pq@guGtZC_*=!8 zLOI9HAAZ2BH_hnJ%oMEu8p8y6ii*<9Hc=*|?h+_=9+Nx_XAqn9z6X`a^v$^f?D_ob zBmtM&DoFeoazSB#&%c3QnSE^cB>rV?i)a8)2QY!tKk zWA*kQXq21uVwMg}$}p?sa|3(gp_k@*L*h#s)_4X2-%9!%Uz!9x3kp2lqH_)+x)^@* z-uEB(7TkYlqH|AKL|fD`CQTosZqZiE$0)OO!hj=1XfHE0-es-ij~EV@Hr%nxZm6x% zAWFq@AO;@7Hs#-3Q%hSx9u%6T-yuscR3vX&ZI8T zxO6--bOrt>|GfC;u0UIIkcHoqJbyDo3@ySYH09(A`t}x#Zh+t8$$rRKKn^pj z;+({r!OD}JY@o7Gd^!;-I(;1a6RJjXS#TSl^tqcKVy07D+3v+>f^Ec3aJ$eQ68Yjv zr!DNVrsc$Dg41P!YR3pZFQdp+JD;0erm@0$-q^06;;Gaiq~t+^5X*jQq@7e|sLhpJ zlcg=`2TwT8Z)=N;6ykbc(s~&(qn~)h>uS*>jhMCDwrp{DdhcOqtdd@;)CWr85=+{| zQn*TM$VCIlQy;yU zfHVg-8#&OC9Ab|R(F#fze+}2S18!pH_PrWv*`=4Yu&wvm^ZB$yimqOCZ!yYX$#CnJ z?>%nvK0P4-3|!y$UvKkSW5tf#_i@xYwWxXz*J{I0`2rb(ZH5uIACx~w6j5E}jHH_Z z>gN_?lJ&?9Ywe+}%7ZRyQWm&S6lm%_4NQE7RJ?~N_ja){Ub;<_@igU+ z<|KK=i8C_IJp^W0VC!EpbhD{TBKqkBY`LRvtQzdYS{%(^up?S*2)!QS-pR)MDzTAk zsn(Vm6l~@o_w(r%&Q}b<3GvZ%W+he_#>>Te=uOF*R-3NdGUGhA|5MHZ-wL=&?P5J*?dxoFz> zSi+wS6D|UsbaD-T*XPm{u`ZT@E>~s}>?@w{yr!+41!y3*oYO1<&3^tb|5^V24?w}i`a!}UaW2r)&! z);d(Z8tU^eb?ei^anncTOz4q3SnIdoWlc{Ni5$Io?Z|d<9wEc^c=x}KkqxeQ80W4N zl?LSu~9|>L_3&=pkwBzn>1s8 zM9sr93}SEub~!Uh6qhMfuHz|__2LJYojVz&Q5U6azwcWcbPh)Eo9uFm0a+40{`KPTJv67o|hC?IXDD`v=R;+p00 z&}4r?N{;j2-yJ!|C|@oe8%g=)&XZWvnAmD)9-EJ4QCSbctlier&R7jK%P*&X^3+vP(Phm^>-yMzNNu{MzERd@#7Rf5Q=ee0884N>4{Y^KwGjJq38)_T_C_Fy{iVB#)1T+rO&@mTpWE!;@iM*xfl8 z{@em_T4+q$9#Zz|e@#)N*eNw)p0>c$`|5#FACrCOYCuEx^h5KnZSh;`Q>qdT`DG9COziToMxYGT-Wzt8E=|CH4KHNbx}Tn0}cF>>^M*B3LeCwK7lwf=iP!4{~2gF6x8^Ru7sPx&#OH1;$bf!`TI-zlDz2g+uvx6KaK z*PNyg`DaTLRC!UIE~Y82jMIZafJGuLXX2E}X*Y293N1-?)PAb)anaa772di{x*$d@ z0g0N1=ffqpB5^!u%gEEfS_%7?V9qvLqpbjg>~^<>cnVXNNr?8FNpv>|!&!;%=P|1} z*y%xvUR`~Xr2!OBIx+{@p-f9|haEh{@zP%}$3+jN3+*|xm<_>kpM>cyUBP)Wlpl`9 zA9WYoQI^z4LN)9i5$lQmQ37M`*I8bj7?g8Gy~mfq+{@J1f10f*yO~S~JkjJsb*dy* zv870HN1*Q(Sa&P>b2Wd;Kil4lKNk;v{rQRhpP0#%l{y;sIb@7gkY8xb@~+q|3uLml zDfm!*+j9Im433NHDdm0*jf=|Im&S&&=H}l_`b!>UbMzEhI1|&-?tr~yv(~h`jj$=o z%Z`kTGW0~1I}=NTmQ3q!6ekFOC!`037Od5PBll8r42;L ziMdcq{F&r(C|9ZfDOe3>B|Ip6yc@9(Y=U?KHjEgu+NTKn%eQhjki*f^=`7(_eXR|fX!FFXu->z#xQYO z82`qZQC&3@_E_H`BwRk0m%RB|Xerl_}yk{U7mv!f|E4oa438w8ZiVq$`RQp3--;qUtghG`gx_5C>t&je!ckXI=5An@+@!vA%!a8YyaEHjqX)>5+C{V=|;is2Zr zW9VwG+>UhrQa}>AH|yyNEV;9JGxUs{e!lBHc8%1>ieaV^Jfh(N6fL#gfv<= z>*!nR7dpkBAP%(An2!9<$%)>4Pm)Ph(6C2s0$GdC#RjvsN1Y6_te&uuKv#mXgy;mq&eDMg{~`1E<2ZJWNfbd*BFO*2?)rXNQyxjLK+Nxv=vvO6Bbwf_eiNy-9}i1Q~;ZLR*3<% zk6R9V(XpP@f|ohwEnbJEcNtBR?Z_VdiaASAJbI-h&M+R$gwmId3Y6tqtO3tS%Z(aKD& zZvJ)0Y%ZKkp#(vXv7b=m z{6c{wF$Ci)SR~A|iz*BXnj%CQFss;E+OkFJPI2HS%*`za zKQEY8?-8M(WD=s_?w#V;{=6V`KRNn&Z2Oo0+l`GgH$)>oOV8F{7bHfQtbLt(l%O3# z?Qai`HeSa>D5C|ZEPh+I!m8QLLxDY8wv^Yl+^(~Hg#3(Jo}UdNh-=(YM9~tNY35x< z7>Da|JK-U@Koq{t42vsPQ1!<4x$lz#OmTKo=p9Wc#~pkYHY2v-2{9%_5_9HfJ%-^K zhAmE@ZXR&i&iGdtf~fNf*{$YVeFx|d^wl3I0=A%$^&b%o*j8}MCkI({r048}?)8~C zDIu8NqI(s6`I6GUhYlyO(s0j1Hk<4KZx9Lk4947J`)O#zGHq+a7{BDwLT)1koOwhr zF4cVW3&{aBY=jwxUc!bk7I{87T(gj`cDV=GejzlRx zg{js~Hzw2V44`{0-RilkvA1x4OP6XnT3wn|y6V&dyr~IMT(%_9afz+mqj}HJ;MjKL z{4wMW?lyFpyNgVB&x1;gESIsHB+0Oa9!Epk7y+6{e zqc*A}KNQILJk?6)e`r);j@A!&RU_8v6U%GRMFi{K83oq~%<_bGM21mr*yHU5g6#Gk zQk%N&Z+Xew4}sJn$p^fS_vUC4I%mrp;x8lDgsk6}mQ$D0#)~dauAXY-b%okrcS=y{ zcYHru{_cF-XD-X)oMDF3C4Srp@!Wij^{T#SeiVMpGHvkP?PwnlfV`WuzJm{#-o1X0 zoynkT9F47w?)0Whd_9tR7puYeIC-2QgDLvBB3IqkeH7bUN2Y4C{5Vs%o;eOqxT|xf zr~15FW7IM6Xbl{9FOFsUrR+{$_?3-+Ga6b)ZwB_N}Z@7DOT%Z~cx&Ly^S zZIFX%v0=xILEJHBKqi;uq-0Ff_gpm%T7j+>u1WJKz|EJQ1{FYbomQ*e)vR1YU>GhV z9Ro zWrM7?XQLurnvQIQ4bZ{MDTF&!Hun?E&TNU$wpQ(*0iM?U^SilnhkDvFcs*OkIM5)# znqpA=83ahGT@is*hwC}gd>nGYFVzDl8kmFLzUwaYMjmv`yYyhMkco{1QR1d4Tn_<0 zs;`pcxORG=&joD)og(irk!LiYXGpI0nkAFz`rE(>%WkV0GKSO);L9dH*(PBhh8#ea zMmoin%2rWlM<%161GXaL>$te!=csz4wh7@dqX~Z7vC!SYaH_*>8H|{lKa>dEIZW74 zY#}BN<~%L=MNAu2hg>TR;z7EyZ89kFOvUq$J4cL5t>XFAJUMXOe9ig60CQ=)4RL4W zy|wWy;8?_AiKJcBZ@qBy{>+|En8rpL3DBn!D`MAx*kByR2to)I)TU8|868)K*ez}h zI|OJ3=Y2xe?5XUU`DH#UP)p+uWQm!eu)agG`V8;+723Txdjkfp31WX(_0A|SDIY4K$hfL;VCd#Cf@nR;+Z`#Z0An^ z1Zc#JUL25j#lE&2AGt=h(ev~i0=96S*8~{!d~Ve5;?6Nulnx7_A<>~y6~ zp^A99ZDv1I#I&IMX21u1;d1AF!^cuB|20DV^x05va9-=owOKbgTSBn>U7q+#yPIm6 z;8NaID63fxr+o-qWnt^QeX+~1NEct@M1~a_Af*zVP7oih4=!j_mxM`A8 zPp=-r96$(bBQ+*WXfYa=^brs4h(U~nyn6v+mo|`tP z{nLzu)WDSgPhv_nv9$6Q><{^$6>CjdW<>THKRMo9u`tl6Eg8xs0a4OWb7@>2cNuDc z`wY%+4baecMlQ}=uk()V3Z~WiJ-9tvtxG$&Zbr?DIRRI!$>~+-K8)8PyE@n@K)`3Y z8f9PN0z)puX@mHZvF6;Gf;XbI8FiR>KMdI3A7qo#UU*7Y&c9jC6-*KqtSh{-Qcy&h zLIBrv>>-B)dm!BUz>QaB*rh9`F1aV{+MJMK%DiW6gFn|64FDF(w2oX;BA`UCg{CX{ z&s2vC96GOW<~%|A;oFsrZ%1px;h|JN#;Mj_4-e5+9@t?WFC2h#x2YomkS2a&wG+{$ z&%*U6IpJHj-hN5Qz%qXFgDpn!M9v>{e@z;l8}&YRHRF~Lt2pT zR*>#)5G16frMpYILpr3pySp3C-1@xF`!b9hD^s;^{4d#j*kJ=pHRmp@!;Xcz6qybEZ5)9N&L zG|>YsLjFmbp2A5qW7NUs(s~scEvl9Sj@)zCBY#Fz>PxO$73&JVF`L&mQ-Tk)z+0y1 zE941;oL}5_^rI)g@R%>rO)Hrh5412k^^jYEHN-RGgy`kb0{dtr@ zN$|Gzoa-tl?H+y;kzY(v?nck`H~&5En3P}byAF5i!*TxSo$;rY+uHdx>rv{Z8Qu(Q zO*YB(QkKz3%N)P#Y4exey)w2W}BHY7r()bQ$4c%P>X?Zn?TerrIKVQviZS7=UJYZcSR`MKde z{GwflW&f3ndEynNuy)H@4wL2V>`+i*wV}<}B%P(JISxIsm+1j+VB(@vkK<#4&DaJj zS&=rMB9wfiY40TJQ^JW3GLkdpT+`Y^^NU6`*LfZtahW^TK|&dUaIbWzT8{UyXYZQP zw#-MpiQTHQxyE!)@nS$kYDP_?0a%o4~`-o{Bqn2Tt3p<+t{}{}L<4w{?Z~9&Sfv6)Jo@_yHRSen`o?fc$6n<0A zCL{l8?ZiHvRCz>s8>wc3;=w#nQW*|k*bNy$1{YRUvi=B>CQhZrGk~q`E@|qbyof-Z zGPDw-#`9xqgO=8&y?(1wN$>%8F%xrppRE`4tk@T`n#LE)oo^&%d}M+fWVe~P4`r^; z7g$^#N1d|>q7P1UaNelD_<_lxh|Kn(^X0SgkMy-foil#z=07#=^bG6UI~8@wgZ(n| z@~hS&JJ`hq2gZe(IA8J(mL_WCEq5r{G)Kg1a1*V zVT%}b$jGm9f@iEalyQYPcQY!;Lc> z(n0ku6R5aB_T6VoKAZmjOciF5W@x&i?vkdB8NdFES2@7T!>;VjvZ1NGk%_e2zN@NkSD0`naZohdD=uO(W2wTcm)jxCi4||k-#Gbf z`uR&+`PumRtm_d`TKo9yuuy*Ajlf6x{V{zGY4ffqfoF~Rox9eozwgKH^mIM~W3%tH zWWE{_XL1GQ6&(3U2^B5x&hq$wv1+Gx#q&$tPOscYJlYzJ<1Q~u((d#PZlM|^exfbfV zk!9#9TB~=tW14DHg7mKZnN#19c#>)aU2!>X%Al_Msd#%Zp1iuRBJ$d6&^yIlln6*N zX_bxzt59AOS2!wI#m1!<~>cG^Hk@0AmmDCJm@yK$YG?RJ!qE(Q++I(JD@M*|`vyHy|<`eWD65blYAr zUHHk%x8ubr+sU-a#!d7QLqGZADb%bA|A}=z0E6-LRzRE5uF6!Ux9DjnOHpyKPQ$I{ zW9gIpce#1hibwI8Hf@_N(}e)}9j#)%dFDN@+QW6B-3tM^1~eX3T1gDmx;X)@V(m@R zlF8=GfX9ISX&t;D+4FLTk;kogeBNxI*z3VWKI-zITm8B{s?dRjGGprL z=Y=Of8~0@XZiUX%8X;B{6NT!YsqR9ZhHT`S~MEEwT@Zy*mz(#c{nQW)h z0!W6WbX$d=+G*+DIsyyZEJr5n1{Iy4d9{lBDw0!lLN7Kiyy}NGU^zoodY|6BZanHx zwm3PDgPszsilkcZ16q$O5?tcJArKY z(W{Ry(j?(G3B!2`QY?NReEzIe_RUZ3kB@7;*>llc)=X(`EPeagopbvtC_GPj#iFmK zFJTX3q#C}w`JPTZ<}!&;Bi<%eZjc%4j~LQ9;@!ef;xi z`PZADL%27~)YHE-f1XQqZ6;C)b(uU`L47+m zz7xHus4asg^%bjS2_1>>0DX+vSDbzMx`HV?)8;@=&SCZ>XpZYKKDYT$Xwcj%?68Z*(xJ|`A?2NU^ z$e?YVM!-fsCLKLFy)t<;P&u0M5_{Dv_V`$wa+=F{GbSo-llWNq^v3&bG)qZ~;j0~J zZ9#5?N)ozj{x$x0_9RX2{j@R{Zj(CERM2BBA}TGDitg3{8+%(82AbFJCOP1{WU8c$(i<&F9T%V4GPwHZjnuD{G8eGJ4W zenbpEtI#zoy?XiVDB!E25OQAQ>Y@Nsi1cz!;c=J7Rtf0LK&z2n;_?KmGH?s#U0AFN zoGz<13ZGJ3kobLvZoCW5#R^LInEAe^i_k2ck(VCLSS9|p!JFK3+p3J?!(w1J@6Ei3 zTQe3zKA-a^ z?h}UM>ctiY*iW-d4sT#gt4qW^n#V&lo5$I1BWfZT;W|r2`xA%ae7NW$@Nqgu-SZ32 zS^`ZQcXw~}Q70VAk2X^L1zaO^`p9&Vmi!n?D)!74K5)3>HpK`DRlPb0O~3oOP9z9sr^0o`DUZ zp{;76z%(2{(2$C{#yr?>TG~20+9#K~AB&r4jf_%`UMa_e^WOTkA@$73sBuDAV|gQ* zS-p5mN;q^eQf?na@bYC&@EM!`XyUI03eEeN_m)Gwhw z4!T4JeOcw11-`Vh#!wt|y`~Lg93$IG#z#ig!#SwC3_xYx#R~w8!*swL0E6~Cj|UtK z7jQHJhg5Fw01Nyzef5n2FWD1|L^lf4agz;ywxm|*h*4DNfR|m~2@Y8K5K}R*1O)kN z+L1n;7q^Lp%1b5s*htBbSba88oHlXn3#p{W>$%gjH84?R;U3N5i}5I6!BSFr-psT{MW|e(NcoO1C0z!>rj<5 z`^lA9QyJ}MZaJ^%wqiB3+oTkcMbOPA-@^I(uY?ER z`J&deBAe87(-)rM?KAoXy2lp@3#bcY0XL=B?5-0s26$I{^(9RWT6Q12I|dZ?K6ned zP5Gx_A=r;$WDk(TK~KSi)r<6C3l_gq+B5NVSnzkR%Mx7PwNFVAo7oiI_k(neGW^ll zqM`8WxLh(gWikrd`;vG(3Yv6#nwG^f0go)vZ#D{=3vJ=YUwL~7R)^Ijd>yTt&3!ks zdtM_Hzp`5Is^_0RLzEXMsX_it47cm-eDHSIDW-SP3F;=yb5JnnqqY{(R#^7E$`TBt zGlY?oM(W^Yf0WALqeib)fzK&2E$nlsqwjHoG*!*K}oiS4;^vS%nuD z($E?S?ahHm*B=&p>t$BPNpxV=wCeq?F_h&`B=kw}1I}_JbfD(UND3AeM!93D@1>`P zIAPuD55rar zM)TYW7u6%JwqkAy%P>xq-^+jIkxp2Abs>^5GaV1Bm^9M^R=#CWdYN0$=my0V^vDp)e)ng~BiNqd< z^?jif1<9CPLgCf0r`h3&Kcd42?>hcfK65`SajWtx?>0_$wh4nUk=5Bdk0!B~N7K+`FV9R@!x0J5x6Af-g~iBXN&A zzuq`3fPduy=bG(llaN7>P^Jdz<5PL=O3T{VpV?#KmNkvX3-3(vdW88|R~A{;7iDKV zcz&t4PL$0ylx`}xL?6&pOzWM3bL@Yr=4PH&MVmZ2&xUDd(EW}-da-y|&AdNn>EJgp2QVpHxg)vCM5>q?t; zcBj=%Zbu#*RnZSjJX`Z)#+Vh;(plk+7y2eC-P1~*lbP>xz1k5S_ExNGTrI!Uj9QWm z5~LV`J5ZSoP3jGhkU0>>xFc6sl4e=D=W7@d`)32ze5{CrlnG!cd5DOHufRFrmCpYsoV|c*pMVQ|7>^D z>t;R86FAsfx#k)xs%if6h~mbwZNjG&ylrwZKW3S|&~w=E@yuA5V7`b))y-8nK;|#T zfqr$}osaxfTvX0R>|cY~p_f#fq|S}Wq-dE~6W5@31k8VHWtqqCQt1rT+0Epc_hDHhC8x4M<8du zunLtc$5{7^BGgSi{&3+UdgA=L%WowS3d?TUDI}j zMw#}Ttb|ryLw#qa&VyN>b)5-FY+E zW|%8#Bx8mm--~cywwa;>U(@QUS`n#KlOODR#U!>hbfP`}p(Z>`-q~(Jh{Pe=0@Uo1 zi-ymzS|@Xej7=salK54FpWo%8+rpK21|4*z!&x4xG?ubo2yb?dT;)$%6u1$yD~L0R zmGO3iwIJw}d=`2PM84#>M86!N;`>yF`^4bra+z1t#m9hL!}kIf&6)8aPvRoX?dV`p z^C_@Yd>%QEBD3k)%&DBu3%Xxf%&Sc`JugPGS6#^KajN|2Mn@fD-N{2?q!WrzT7J0I zhf-}!>f82Q%((iN;ZDF?N{esX#flBwF=^cH5QULe;Z!ZWP*-_jtOw;6U(!Fz!djc0bGcFD6dT9ldcu_*d3aH{Y%3Sa zV)MvhhVF-d=e70WPCiGiVFgq%&IgE{iPh8uTKJM1C1wj1>QuiqQ|*QfS2X78*2>k_ zgFo~qI9@YO9yLRNeJf(#FIWHSl$dK&(NpALbajtaGR9xjZ<9%roVJ__->cc;I%JsN z6F;ioRA(L=u4hX7ssI!2`Bji9PvmIa9{q{$t zpj&%Vve@BQz@oJd%rLaZoR1g~V6D@Yv@=6@ifW2`VUE*Ww$P=;E>%2tY>hGt&)q&3 zTFydgs!i|kWmO%D^yT9T>uE%SNHc`14W&Sv8M-Phl@TF5*guBP>gP%hX6P_?%hf3w zfz3UAar-lLN!_pQ{Wr_iIn5Hq0ab)z{e_l%zML~AYE9?^o06pu&FM3Ab#D_wG|aj{ zx38pK@3tdUB^M^R5*%?{#|a;<7bW2^a60kt5DNXHsC0PSAIX*>j=O_%J>(4pDSJMt}7sB@Ng zDgbh~KG&ec4Ya9C!Pt((oD3vekU#Ju>PmuX;h?G^@7gcnqYW9j6lE=w1F%N)v6kW}sOk=x5Wwe|L+XxG{NjaJyg_U{b)(Uig?zQ zNB(i@Bv&`9jEcF~Yosd!{jsQQ+qVHe!=jfkcva^mVK8)PsHnFg`0MV=p5(f7Mri6)huC*B^1$E?bq;QrDMX92M4= zPXjt(4njpK8#!>2#STM7b6sk?GL(wafx>CEXh}K~y1qL%V>%>OAS@}WE29;tsQv;5 zjI-f*EYGS%lLd5TXk`N%jfe??Z7+oMx0}UHCy?o+~#6m{Z@CfhE4A zhBcL#U$zZfEA#^E2n@@?{6~Y`$Ya*<);}tM9r9@a;xfeUX*De4NwrFPm;Y-Yv_sJQ z){ERcFCkF}pCKGkZQ&9GpK1db%})P;VXF)=6m<7o7*>nMz#s^gv0cO1uycuAW3!?Q z@&bOa4Fd*~)0@yqnxKS=a>Wg+oqJzGE8}_qaLIxgDk^HJ2Vi^y;vb&? znUFS%fqz6%j+6Jv)=EWtEfctq6A%O=5&V-8=>M-@$3JgDY~1$M#qtr?mxun3myCg- z&OkpaD-L-AD^3i{{)}4m9W%h){E;BIPyN=MuFnIXAh>#CL=8)bh%i`>tmB_h;LZPp zfYv?Y(R0=s6A&shz`PR%Kxd4mFfGIh)f3afFA8);R*YfMbm{f$6b0E+m!TwLL`ClA;Rw=-201N|oj=BX1 zZ`L=ao1FxqFhvcbR~y8GUNNwg?>zV!hSM1w5}7&OQY8WJI~ zJO+aD%WHNVIN$&Uevm)We#3t);!bA>xc1h3xda#pnlbnns?lYju`rqqCx-DDSeQ%* zq8WJf^do=^aX;p+?RPs55$URp|AzKR^(HVS_8*;)3IO@sBmlAo!W)F!pqjvcqaoMK zeu0rz+A$!_0U(I%YI=wV*#G((js!#rFdPzgH+~?bAYl@bZ1gYS0nJp+>i=Xy{M8u; zkqfwn)SKu&NH!z^vpFO{l;FC744(p`ULyl%_9+_E(NuVXued=p+@OGba_{_4CjQ6S z+rHeM$G&mtuYnJLiUTkI^>7F%A(*Wv-yj;a#)j2cm$o9_L#|kP(ZVmF1tmTvb!D)Z zcSvVD81Eg3oalC^dd75RT-+NF22ZkB(IMFiPO90rXA!zs>O-5OVm?g-2JZ`<{_nms`;8&2}0-;l!P^+l2-R8CjMi=peTjODtI zAilumsSm*MV$r0a{L0kQDz;I{jY5>c$osC916T2fMu945J@w9(U<<`5RFt&#!+zwy zh=cGvN{>STHnG4H|6zUS4+v98qSq;;mu**imO#LCWw>{P5CUls{R|Kg{doiI3Fm+g zZ{Ih84&0{?zOqXX#bs2P-$3Arl>=@eQLY{K0hCq{8i61R|EB_8Xa393ybdam3|$GJ z?`?@NsN%`Sro}G$@;{rvNJyooum#b5*Z}mE)Ih>@@?S{^_zV)-2*kyG8_N0Z{v{cWO+1J=NPzq;)4pT?E21_pU^$PYDi&lSWIDsQlLnOzQUPRv0XwkAIg^0g z&IFJoH~ev!udL`KL2_mXGyR*;lC4!G#ffHy0QAmbHKn9L2t{fCTk)_;x`Ejx)BiEn zI~ABNa|9{eI%G8WAbJ-?r_}7B=VLG|NB>T{3f%6&pwBZ%$FLdT6IFVk&W@uJePark z*H;6g9a4BVF=)yYIsW<>lAJryAZSJ-BK{SbQEdH29Zq5SMb>09V*A03t;|o@M|ac&-2>Cje48CJ>b#2O!}N1h!@5MLt9-sb{s!nFfmih z0*?O9#mS&{-NS>u-HinF2d}}ary8WU4{_c^-^(ak>7P=@ zzu=Xso1R+y{)6BB_&znkeQxdu?J;EhOG${Vw7>4#eoj;=iQL1vZ%-rny%tqaZ%i3e z4AkX{34+e}vpr2sPsKEkx!*cC&n-J|vL?u`Ozdk>U(|;0uqBe@c(nIZpxOF}6npzZ}O4_-wNld_hb-BWd6G7hXgJzhUu z_gq-`?@Z1YLO?5xrZIMhOdOlUhwh4IjrpnZ^9h2UXhFRPfl0N!) z9Rxifr`~RPCND8xYr;S6mY#tvJu~Oy8X5(91YU;raQK34MzhzqBVqgp$UOt6U^VtY z%lZ1Pi6`g){a{0u?wQJhGak?QsU4<4WKTz2#%``+vn|>%voPJSP}dw!-8Bw7FU#i6 ztC+0P1y`%leTl*1xqJg^$K`Qn1szu&b&OJ^f}19An`MO7OrfI3wxv0->ohanfDGB2_UX@}5e6)O+9H!ARE* zxvk!Z_0d9`dKia~sZC(I&T#kQ`JZ8aLJGE?B>P%wRhIv}x!8g}#_CtmmdCfvQZrY+ z#Yt{0;Z=qUBWTrJsKtHm$p=sTGy)me0>0E=kpx?IR9%7+M%)A)hFc7akR?8DHC^&CJX}P8@KpHA~&X&>@+$OJAIl$_@7`pY) z%XC}*iWc=QRd$cmyP)vp+2p~-uxvA0mYCNFU-3XQ>;ibJP{9iIdp73g5@E})jNKk- z$2)TcW}>M$@|Sx~^_RGqIu7Z<`#FTy`VpuvlG=PaG0vmg(HJS$U#AsLu&I+D<(LO7 z&p9$O;7Z5}WtXw!x2c;znZFLp$T-lt~2l=a{Hw?<{lDH}aAa(UxF62$p?o4=mu}G_)ro z()mUCY|lb{@#Mo&(=~{Em8R*;@~rFwQ}JzUq+|Ecg5x?E+>l~mmOYd z9wY(uL|uYrLNXWZNBc(LZNE4-on!SCB@$ARD?~v|!#?l5z#1n#GB541Zx-o>#q3Ay z7h`(kHWr0DoL)+#1yl3LIULavFIp299NJ^7pDYreOLdjl3iG`T!{+Mv_$P-uq@YWE zj(Fu)e_30Jh^5ZVQj%DNE6>Fl|w0o&=p8GIcaWj?~&U9z|oHwH!j8 z8F7cC5(Yeoh`nx^np{Uo#D@}0T#%hxW=CJT*qo3nJF70du1x6`4E~5^EI^#VS(T!|LD(<1IrTSjdIz01_?mi4(O0&)-qZ9jSPvK$< z+TPf$M^^+~MLGABOCCveg}aeD4U>wT*E7oL(94LgDw$vrYP})orLY5%7+KN)Ek?BA zRck8>)90IIb9KH<44ENZ_hxOPZolvbk>BZD>=_{HwQdY?S?RoKeM-P&!avXDdg=AY z>Y;(+axU&@#FJ}YU-zsh2AIb$7ri@(u+v1lk$hAnq8|8 z_w<1+fJP#1Gq8++Lk|pWaFH_svH%YR_-Q~pz7JbC6MRp*`l2!INtB|4QkUG<+?$%mf=` zg$0HH$;Wumeo7rc10B%#uitjr=Idd7ph83#0sJ~f-}dc)?(Qpi*7UZ44$YfHlLR{1 zy5+8#7nXZv5ZniM=5r^}1v4elRc7#7E=?T)%lU;jbm(ijKRf_v_-nb0*RV*e$Y2}} zG#r6&n&*T2Tm7~|2KyJ@wn(pG`{Ja4EwO%rb>?74ROa7~ch`RrjR)_3|H3@M55WubBSgg+K=%cc-klIWNT#qQCj0Q&Hjw zrMU;bg$>I7{-6u;BXlN;_YlMq65p!ElYXlI1~u^gh2Dx2E(ua;Fe0|+ePG(1XV z@6Ee7o(8PK>+lDqby6=%Pj+*a8IN~DhhxQQ#+OY{p?S$U*`bUr)0d4#x-vYKChyP9 zM7fGDVT;+gO2d2_t@033P;vFMMeeGWE1BM1`pqKhB`=DX{A|7Yarvwv4ZO0+FufPCFA5064`n&t zO6$M;icA(#WtbneL2lf*FQmb1zgo>*na_Oe7fpFP%+??HVUPShl_YqYyN*5NM_ zRa)vM>7lrb_JUv)ldMQFlD8tCO%&`n$4MU;gxC9!VusB+mvg+cRnITo-a|KO<264h zkzajwtwRv{#+jMHlBV%lc74)XbvfsfELTw;tmB%cIn?q)h>TKFD(GK7BxHZ{gpyLTx{iYduOEUt*`axXNK(rdph1 zY*cpeS!yYE*h+0)myIF$+5X`D1a7<|J^y>4*yOZhg?~@~t;OJ2HSYHbsn4hgD)XbqAA`f8n%SS`c~v2pOXz`R}&ri_$w#3 zDJl%Jhw%?AB954NQKxA7JAOaq$GCdqoanW0WykD!o0_=|>Q6atUdgIYox4p+&lSK- zOtib1y=P6!F73`iNn&5s7`jF_^U!F-mx0Z131qC8-o>WXlVXhec`j)6VNv}QHMX)d z5!qBL{p-f_j9Cr2rf8i&2cfAEKAX0qrHI`*JoUY?7-C(q1Vx5f>*GfT^PV6Kvo{R$&?)rT0F}*gB&rtyDEE7FCemk)tFy%6&B{A0 z0cD~LUsW6^pDj!W3$B}tu%BotTR5sz^=1`0SPj{S7rEJ_MdvC8r{Cp{i0LHB3Kl2U zOi0HpH54^cyT{`&(+9gK9G1GVf%g$A72?Wno0c38Sz$Kgv#gT1jJdW57LFJCi7WT7 zD!$1!`lu}^)$zacav!9f%iJx`fHXWL}Yq2V*c=O-+~R$v>yvZw|=dUY+M$rD;1}K+imV5}>8_LOR#!R-U{~F^S-=u}m%zT{A4t zL9Xctsg_r8YzsE^M+>8SEwImx60N0zuX{+T5g+F!%*{kx|KzeEqUqzA+EM2h(?Ror zd|@K9K4;XmOJG4zX6-Xl12@Ev^x|!M4%`|U0! zI^lUiXj-7g%@*R)#bCEX!6fU~+VNycno`=?AEuuS6q$&UeN?2{y6+F0ngg@8MRgZn zYDmcAtZZ`I-0;$vX|H22@u8H5&)Q|(S&H^)nansG{=k+@(f@v8ccefhf`wA|{>(Tc z=mt}ReoD0f9bZSaXmmF+Og?5pW`?lkH2gQupug+j}76a^jXiE(SmhlzrG zb!&;>`d*Xg8hO6#jInXmGRf-~e}_j@J70jNriO{TQORAo<=h+RsL|6R=GVkNuhje& ze83Bj5a;f8CdJ6@Ml4aE^!O0VlsdnD3co>1d(c1@;|Y745zG7jnd$_RmYd@z&nuGV z0j7NZ-(xyqRwfATK`_c&Bl7uCX;BF&Vv%q%Z@J#htRz*va^{QW>L=GRL*6gcagjSA zruVz9ob=8m6E;hBuEgJy+QY>2R12mLb06kuMULlm{7TZui8SVSLV!=(daNi{YxhvrhA{q~;VR)M;C`_wAbowkVO(khF;0A6 zh@YOlrV$xDvl`{j`F!lHEOFc|vYyG#v@z#z`4SudZ6)u5CXsJeCN&PmXNsLc$n$~7 z@JnSU6x6t4>Yr2^oBt+`3+kJz1A9Q702V1lQ-}Lkv+o3WYQyOK2 zfcB1pxIxnAQsU>Z;Z{P($jNHzq~W_kht6`r(Z`58^~3WwZVRx!#%10Fg+GSXDUp@<7>&FJ2kD?hvYb&?NgC0dr@a`BwNCO7 zno$h871yyuRLJf~WS=E#CN%gA(Kne2+;HK}XICQy!dTF`~*eH4k61 zfVloBjNhvh&|aSCV%iQFzi*6) zLg4#4L*dMCi#iwbLM4Q6%YrS64ym^JE z?agbw*Fjb1B@q~cFAKB3dKH{j)FwVJk#QvRb-}c%!XROwUs!9ilT5Z@9 z@fZv%LFVnPjm&YeGj?)b_e;nEZX1CTIxG=wunqE04xC*c@!d5TTXKp zC56Chi%1=T7>9mJ2n7EduN5B!q|dW>OZ7Zp*ZR%}gJw4lTSOZT5r)@;=^;|z6=`h| zT}a>o-amJ~Qnf<<5Qf$lF+kf;VwLxZhYKp2XtE4X#H`?L#|av3z}vDUHRcKal`4D zal-;Y9uXICi6M~UzKj7KswI^6pJC4He|Hz(U@)Z5bEjwtHM=ZhloYwXtn;O%HR1)< z<$43#*D#>NwZMIxE~ymzEa)YEl65I8HjN|mN-{1FwnGwM*RP`5e}$>K+VnN z`RgXz0B45R916=Ar^WzPkWL$UBU(pzAUEDKMjy;LYn<8NUUf(cT#*CBe^0Y-XIR+z9nti2ukhOr>ZFm5Ga;@*!ShCdj2(q{zW(fe5DJ;Y5 zMP85&9Dv6yr)3Q8DZsR?042at3LW=Nt7uO^|OHR=$8uI*S5v+;Ds{a4mj5 z*zhgX{l5qQ>tW3dy8Yb@L%MB4^#1=-4#xBRXQ;Jnz1=@}e}O=P97w~g&-sh;|4{T# z$G|sWU}zxNq#QzeY;FyF`-k$c`oCt&{MYv6nVs5y211Da{|3mWOSy)zsC&(h!~d4E zt6jL(_iFbT`dLlB&VJC1Cbcp}{i{MrK7kNVrX1nsr6zL)X=|iAsFpw5b#xz}<1Aay z&Fnl6n z-UOn5g_TpG5Xei!A|Eb`4;`bLq(8np^#~@w|53o~+z{8+ye_)ikBX(wdhvTumP}lC zvIh1-V-tn{Q@V+6NXmJ z@JRguie4dC_Th&S#`j8lGtk;MI@~Q2TJ+XK{U{`zihehU_P8|EQ-SZFhxch*H4Hl6 zKI&9y>y;nj)T+I|UF@H6EHPnpM80n!zC~`nEm(KI+;Ph`FTC~FwmQzNGlVqtr;&^(h9KSOI4b%+P6`Ow?zSu>=BP)R%Vu!AotPQj zxm#MJ_=GWk(VAbPqopIVz%pl>Th@yn?p!d2Nr!{;U}x{|l>&2LigKF>bDyYkmNGBxRfx5gkP@kNMU)2XOS+-C#3`?wRz zo7tzetD3-;KQVKCXiu!l&3_zwcdg<)!gn8!?;08(nJ2$9RNZ-bZU>BbAIaJsqv_s% zy9_CPlr4V5NHy9xd7zDybV0#&dty$(^gu+_Gj>J}Hp8i6p+cnaEP?sv9B58i-A*$*qJL0TIN)b2Dm;#D7-)UW9wW1h0|A!V8p zZmzC1vG^R^tJjGa-=J&0Xjf*xL3!=MWKH)+;F%NSjiR8`)~4%uq~9phi!IgXh01gZ z$5+zdmQIyJpPhys%|_?acp@gA%{~u~@kbA(V9sVRU^v=*s%H;1Vbx?|N^GqL7s|9I zg^aVcsJd~M7}8D;w!=_mIeb%;s-tP+`zRk=whW3H8EbVqB94=59ck)|5NX6(2KK*s zjv^;ZNB3H%Y^K8$e5C$^PQ(X);_YNOVG=6PD!)r~G^R1}Y-2Ww+dtusqnTR%neNIj zc+KS*y1od$m&9Z<+suX?V&j~e#YZ)=VXs@z8ZEjMdH>kJ8)COCa;0ytyba;N*Ix!V zIK4DzgxUTiO1$~L;ho#;9JfDn&F}l&^t$$YDZv;#WTgdV@R-)<(MD>ss)NCwRWY6x zZ19{GLj|FkhnL++A#Yb}v*iM&DtM)>X z;crj9`iT5Y;l`(#ez3lNijd-LUN`WRO`;vpO8XQx>gVZAd9;|DDOyuL*toW8!M(BL z_lRpvaDl$TpEiWlpnX5w`8GwI^7ywbAAK51WxuJ_`o3HQ{Dqs0EL!g(TN1Tx&iK1O zGE`68{;o`ZED*kK&%N1!nTKicf?r;aI}|a$RzNU= z=*B08gX;!Q*>}GV-oi~Kv(EJsu+fXe_;uZ|S_FQ)|K$?I@Z%`o$5nm!Gp164;llRh z98W)DSLRW9$x!)I$drqzpaNSM!rRRvElIu)8=qd2s$F7TguYe%8c|$qS@-1obyqIJ z%Hks%-vZ)Gyt;r;YZ+$2KaBEpn*kmvp5-@p~dpct(96o&0zwZVMDXI3RCG2do( z#D6gD>u4yI_KeYTAT7-c1!8KrMahKs*IVEWAbBlSdXtz`s)nmJt=^F;TiyB0N$Zow zzfiRjKtddGG!}IEG^)XU@$664WrfYy_pWqWbC&&jXx%kDN3|)sT9nv&2`eh*2eiU|J^dg=VWs>5gwXRh^<{(L7FqI+N)JN zPS``NTPEIS>^G4*{g`k|UPvQx?2`4x^($)wtmVPQDSbui%?F+ABs0n&6rTDT@+K=6 zoUP8q3C9LOrvl~KcQn<4Z1V_*Pm5M7Qxg@;bf_P z01@G01EmqI=3PF>MW{E78C z;+@)tXJ?-%hX!3gZiC>A9$qHZlCEPVk(^+Rrej282Ym~D?ba1$kzL_QupJ=HbrlgykEGOD6fH_bSRSRzjLmHs z*-|k^*aB@PZyX)T8@Kv~FCod6Xg8a-T{c(2Og)L!NvSOL7ZhnMhqCC>^NoLWt^0Vx z(wRE5BALdypayc4-bSF0x!WS_NEZ4RE;zkilJbjJ7fs~XKDVT+-xxpAbHp$1G`!O3 zYh#H2P`pI;)w0dU-+wY}o`mllzHtW+rbhFbfvgAiuNKp|C83D*WF-YkWzQJJ0K!BX zh32|j63abnY&KuzU8nW1*&1w%tyK5`AN6=Vbwo{GRo|~+cZG(-z460uJMk{PPee;? z^!Y0>cS2B`&TcZooAdIQCn7ZITz~X5o%7i6NqW-;;N&?{P?+kTmZuCHhdK_WbWa_c zUB78A>_(H;myKbaKKVuZo#w_-Qw}edmpsEZ;j=mVR?kUO2tn1^PC;`VA$B@ObT|I6 z@RDfPI}~`*U&+z6HvI!;clzod#og>Df?GnEj?+i@G;T{yj1^0ZOQMebi5}Tj)=;n6 z9ySR*PmYS+=2E~ctUZx<$iqKW=x}t#(`@24kFV>N!Y$_@`CIZJSMjLTlm3wZg}a1~ zNp%z|9~V`4A6kxi9*0rdX)$Z+-bp11x`>}u%n|b9!Q?3Y+-EBf%926KeI|oxC}n>d zI}K%FPOLwvV}r|x9I~0D_=|X(QC`zc3?0M1E{UYo(}|~laX+S?{m>Kd{(b$GYGr<${upwn(`rK62P@z2R9z#TUo2}lr#XdV4!rv` zHRk&Snl3Sahc}*o@BaxN7OS^xI65S7hAIe?_!4PqSR}s1$bOvBAe~Yj@nK+UCe&qr z7S4U7lz*^Sm}lXRLu4?|6a}b_(f(Apl}bW;AWpJmkRrY}x@`3SQ1%vJRebNiC@Cls zg0z50Nw;)&cbC%L-3^k0gmi~=3epYINOyNPl6QgMUz~IAInVz-&%mCUwdQ?4Z++I@ zv-esvUv)5W8wj291UB(&W%=&}cUjvHQ@*;8S6HiEb2vZ6!mxe!&ag~_=YH9GrUT%fm=AR@*~8sv(W>P3sw&JIPRW_&4Wr8vG~;NUR#t2t?nyzz#~u&2hBJ>`viw-wh+SNkKe zh$PwLwnEBlpWMzEsBNnZEu|8TKtI@*`$w3p{nn#9GrFp^HsLP0 zr60hdIbo5s4|VBunAFG|!69WkcVoVeN2P3tuIg=SgS_|20jxr0%lLH&qn16|S^j&3 zkYi<+a$rfc6>(IWnqTz+;)zTRe(aphJF>++bpT5tc z=i|7lBHv|FG~aPnxT?&FTr+>DffLrL8*q8$W0u3zKO>!rSWf+E{fecbDsI4zi>5J4 zT4vFxaDP*3h?31_^$!&pOSVgfj_Dji6WXLN0xM+4+8qSDB?1N)j48LqUV=c@3tR>? zB`COil3L+i@yXA;%tR<|LZ@inrkQw$Z|XNFAQ!7bdP=5~FI^9nC~u`O5E-gtyy&KR zh*a!c>+I|GRPetv=xLIv@i~_k3a3*41}mJoth2BnWgCquI9DdUQ-9-d&$}zfymVt2 zbtPn{;w>)uEw5WwXL(GkZ+1vGN^6DUl@jxITi&>Rke5b#hM>@Ko4|tq!nv49SSDyme0x^M8(cc7f|McXs?SRU-{ z;s!IaA=B0EU)0!YDPRly;)=i(_#V?h+z|SQov<}&VEXn)-GuAG%iG-~clz)0_L4gN z2Mr>TG6MXPfim+dClMkw4ugZpQEGjIP1nVrcdMKVf6wQHDE&%GZ6Z-`W~Z3{PTbQhR%3xEnL4RkV60OWOklsI0ji$`d?QbmJX`5DP@t2iu(?t}X2u zgj(sLF-5J?KRX&aRT_s~p%dD}5JZs(WT~b5DnmAQ{gqQj3iPnF*-NE`F&y>^@#924=#|4g_%+1%nPu;Sf}IkIxibz*c8@+} zLSd<`<`s9#upu{^E!CT=Q^i%rg;0&fq83N9#9XD#ze7@sG?k+h5VGwLJe9e8fgp{r zdGhL%>p?TORibIr%e+gRoQGUb2=4xVf59j^Mm}l%`%-f_&)qw0%AC6bY|8Z>MrF))Q(vV$|!DxB%kx$*?J7s=k!-yF6L}-14Zl z6F|!#=Mzv!9N_w$C-*BG&NZz20`hdBGKk@~f?R$z~!D()Hp}tw;4yv!OkmJ}0-d$zyOUCH@9ewml#Wy+l5`JUZ=e?1fP( zfC_wS44%B0Kcsrhnsb)SsC>J4SH#Kvlq9jX8`zXBdLS?WZMxb&um1@U3IM^#{)l|{ zgHh2{^UZs*d#$+g6&r%P7$;}Lf2Rbj88)M`en1+C6b^_~lUBc~Pj**Wtfr?+Wz$I* z`U>F_2*^Z(e`T8ZPjGsPUN1)zX#HpZ5ody2jY0{?F(Qz)gG9en$yMUYFd+i1}klxuGUKy5^X#bRHQiGl>>j0Wh~LBjcSGAh^p z22V=+Spjm#0Kn>HRDO@}*W5>+C)m`Z%Uf7J7Kn7L(UTw7ftdf^H?-F;>c1M^ zQhNf6UZM~ULX{@A((Tgj!cxBxhW>E*XK@{y5~z*>jx*eIE$U`@bD2@dyE`nlP9LyGqc`P`>RUx@;e6x3jq|f=iljUhd&L-dzSsz4IpzcmAk7)4n#%kZ{9yz$Iql+G{!un3W?9Cyb}!) z6;$lw>)#px?`)e@nL+s;dtH_b-q!z>0S^fEU$h8oh5@hw;`JPPq?og>JVHR?+KJe& z`M;_Dn0((gfTcBm6Ok}c|0#+9(_?O-uh8h_l-PBJJk&p?&3`f-)aMf&P5y^;{$y%c$avb%he15c(rB6_UsCrkfR*^k2j{<(7Pzpgw?DmF_6*>YGJ z`&ihI&5v0JDioIT4WsfJZYQ9=?Y~^c+XW)}*a8fCENNL!;MGexkV*g4_tX%eBsU%q z$DgPS%GN&s6WqGTedYM;+Rs+MNOMD*K{gjE^AcKhhl2ktth!jW} z{WMU{x6hYAO}hf?QGwwn9)oBfhyI7Q7_;y z38Wy`-^d^M{QgUj2akmW3d-_tL9NLB4e~o89x&E{(JY`637gWr-PmRyRH+KBVA;># zgkX3S`1;Rin$Zu)46tM@cFkXw{I`P{42!IVq5%o!8v)z?QwfjbReyf{?xSWAAUs(l zy1Y#duqBva)b(N#(8E)Wq#A(ie#NMK58l8mQs}Ld80bGt@vK1xPjxYs-5{ek)ZD&tzu_8^4bKq!0XxQnQnhYCMXM53W2{L zZ2Kgxzp#8r^fe)%{mjT`2hoMUzElU{2YCstBaRURBu4cdKC9k~hE7POQU+Nm9moPj z!}R(QJn!8U+*<1d9@%86@BQx3$6XLgHE{wt#%b&~xp_%)32YVUS>{^&boBzWlF%y8 zCYmlRCj>i1HhIhq)7Vu*ys~Q#=@(0c@NQ63Mn*VnmA6`b9l&cPA-f_)HGB% zv$+0uIw#wgax!u+EWUH#*RDPHgivCEmwR;`UP4^A>a+9g&l=YCX-iUU(hT)6az6T- z@i!#W=kpppcuE=4Dpjl+_};(LUJL z1s$rzxSB(=pZD%vtE)($t<@Xzy%Y`Zy$M%8{zS`CXqdi*C70?P!Jyd!{VPR(v03aV zBm1^DWtO*HOg@qv+a0;dN^!fGO*pL@wbU1~;45-A%#6sRFwL%;mx|wm*-w4U2plq_ zNqfGAx%gHf69>Q3o8w`%zvf)CXJa@JQ z|8SmT$%%w?-RY}wNl~`DWPi^~t`^k0{rm$P``YH752Nd|M%VKke6Q3ZRLD=$0>`$; zZ@7yD;petCEsYwEWUi`6ua33!OJ%kU;Htwuu@=JG?v;|2yrX#!8^vL(U}6)yVDpHww`(F@jvqjsEgZ%V$`QNNhD*dTqe%w$;gYXX9lqxG(j z#l4bXdaT)9VEkZi@$;^jOam?(OA|syv~v^^LQ!qkscLFmD$ABucyAJror_@+x7CTMwG8uLA z2{ee_lgV+|hWw%%DL9nB-;!gQK#!Y3E{!$Q+0njS^}!5>2xSf7zPTo?hI2{fPCe8N zgVUz|-A`>g`FvvOYqm*ArV^g8rY5c4xpH$bwWxvAjz4Yy{zqB4ohxcX^;UyOG+#`E)VUhWNlMNjYaS$Bdx@OM}6*ZILbmai1)FU7%)q&dR0 z-Wt(+gCDF&5RE-xZS@b?BrIhgGJYm1LHF$_Zej1{-|Q18f>U~)*LXkM73BP4QH z>PhFMSYV& zt^lkg>C^PlLw~M>zwV6yXB6IxSJ(76_*qi?Jwo2$(c`??xp=i3abw{YvGFw zZOVS<&-Equ2~nn|QpAR)q!h)NM3<>fjbu^fNh@}3U75V5pICbBi6See5;INTBBJ;k z{alM_uv2RQH&5}pvT!fsd9dJ+-M7kmEdjYssl^eVaISVl;z`c3{_ISG>XdIkU)Fet z8Z>xd+J%wcw3@LMF!%1G=bukPxPE`-X`-I_>)UnuGjPL7%f{^(N*vw=)>bTzDEWjk zhD3Xkghg^u&ex;iL;@nwVPDbfqo--p+-`OYa88lE{Pxzv|z&Cj=&4B>4Dm zRln*=b7W`-E0rT*B7}ZgsekX}X!O}@JI@>EbH&~YceJOx!8bWN^(3w4&~=Xx1^x)Q z3Deigc%_Q!pXRcX<~yn@R$G#k6z7;rrAkU)&PeQwC9c}iV`ykDGDOik zT#Uq-%Of7gMILZ6D>jE(XWxx-P-$HW>+6>wUY?GOS(Ld+!ET^tEA+n>>-jWqaNK^3 zSPhpUQor!WeMDqTF$b}|2K#QEqp-ZB9_Q!dZ;nl&PMIUYtVo-C`p7XWg2hd*Wh)Y- zc-#Wqw-EO_D>d3@He%m%`Vs0WAJZIujxy>42GpsC1YB%3r5ay(rgTXsQiBn*NFk}QEEOjPy` zPwp+0n?LCSdq&bv-5}jF!8LwJQA1IcJa(Ynz#cw53W+`=0%_UCqZ|%T7b+QddesjJ zlIb_0x1l}!QWR2o8a5u3v2alBBvpn0L^!wr96M(Vpdi`@!_91?pwS?lr3*N>jN%{B zGO~C0bFlk2bbdu@WkLGewGew7rJ_rQ9i~}?Kd5Cg)E-{cEL&iW;U!SNZXbNt3n0dh z5MEH`2|z?X!70~a0~&;brb@$}fc(?+)D>-K3k6wo8$}TM2^0kWHi~;ZFX)<#)avlU z9=^mhUErvm*dE^Vls72dABMeym?Z)!st;{w1SqhM@Ih*2d-kLmK5Bvt&)4qxUn`L3 zeDzg?gO<2Ow@{KDfX5yWT!EZFZ8Gfs@q2th>H1{$@Z##}0 zkW@2b5f0CnH~rDej}G^B{=1v`9FKpn4B)fop1rDzb`rh*3VI8+=I&uv}{%@53 zE8t)^xZ<%T>c7*2k7<}NnAAS_#x?pk;nj!dc$rje|Nes-GO~vcBiKgaCHi-`RDDE) zmRO{>9^K+=i@Jqk@M{|-rW!;`9|VTD`!QNIqMrDkg!pZ=yn0u##u%PIhrScvLNNvE z^d<)Bcy}Km3}bmhCI*1R03GkHqdw^Ut|RCgkDrFinEgZu&DIlyxAGHIx8SBp8?EXt zmM6Ic#^3(YPyIthJg&gnYC!bMW@|^WE7(KSc3S7tzcRsRv0LBA6TFp25^7+{IG~F# z(O>7ip4e4@GkmiwKmS$0Ns_&_KB|d6=Ls!&b70| z`SC?&WQ`($)pza%?X%;!oyYdsQ0@i!r*^lIZnJ_&z*?oW*UnPs$B8?u|BEm{#Q$eu z|9>U#3q8RAnj8P8FduEoy>+>jO;K+qCszQ0GrvzB+!D3JkS%iQSgtd;BNcEkm#2qkO-}gvjvXA{}Nq` zsW7{L(J|oF!>D1xqw{xoP`?3L{CrvI{c+%lVxWw(1?J+TV~`*@0Y>2`b&#A{0%M6I z|2ceWc_g9^Xccyt5#At($5s(`m_-u67J%?e6w(E(l>osll4J-RCBd=7kPr_OlK!-@ z57q>qf+hgX0{(>A#M(lMsq#l6{C%A*U`^wLbj-^Eh|TOvm@&AX0PyA@`!L~EAfVfG zz^EqyY4-56(|$;DW`G;BVcB8Ux&G3Ix*a>rF@OPVsor?vmR2=5cK;r!YynveC_E?~ z(%*y!XyiHn8XNu&RWd=&OB5@<++Z9)d%8X!OqoTIdg7EvKR|$xpwV<6`XTYX1$qEr z;QL3ee%u`au#KjGCq1yYQ0C&F<8kElr#)~`v73@2w zr+|>$B|if&lePGR9*3PV8(^H0I7owSnZ{v!5Mly*cw|XO{=RLL$XDwkAb*ZP{Qo-` zPu>@_cpTjT#{Xp~g$K-@`2%>jMr;eE8jOoGU@U*^`9ERM$80hHbpJlV2Z{byd6vN1 z33YT@kWO&WznRifO#F#WbMRfRV4Yt!LjdAQSG!u*PX0@*_*-;=kyN+*O#V9HmiR5W zck^^fOW@1|FYX7 zL+?aAF)qRg@hv|#=-2_M5ET{B;3DyDo72m`eC7aXdd>l`967===D{pz@c`TpSaaRy z0+2hnfeznnjSqMl0Dq!iGr-43TDn)TIxK)63I4VKlj5H{EXTu@l<yg)l1zfPq9W2k`QY!HwTRm{@IV$a_70}# zECI=fwngy>FgPNkcqbs)6kci)5Paw6miKUP*ji;A?=~uk9K!T)&%M5qJU<4%oEcIi zNCWLUXU7RUkL{bG+}sLJ?d~JpA_b9=iUb6-*H@zF$5A`0t3v-JjOhO+?EkOiQ=umq zKr`U^{~=6ZQ}*?h^MAqoH-UOlvUecheX@K7vj{7ml<9HfkRb=5zf=2`i>Fj+*wZQ} zmnW#*LrwX6S;*4*-Na3JR`ppi?QjatD2KUg7}l?|A6odD6E}Pp3ZiiLK6ri~{7vlL zJAU1CP)0mYx;h|BEv7`U>6Ni0bK~2+9XhFz}-fromg*D3K+k||AF;1fpcrrdg z-`Ji@Cg^Pp@tHw;aKFqz5rqp`z`gW50!T$C^k$fMA00ym* zn(zEfV2Mn8yUuY}I1-!dcj&_6fyodxMKM3BHCQIU zJ=}L2oG$Xsl>OQ43w4>Ge=t$$0?UHRUqjpCeOVdakr8zm_G$KlzJc+&UP+4xCoRA6 zyRB)yKb`B=6L34#p&eLW_`d!?d#E0*-yOAjpYm=}J?7QzDwdx57DV;75Ld$XJ?;n> zC-+B&dXjL%etmU^Bp!@tm{wl@AQ4`+q`Rp~c*u7sBs4GX8JEsD_+M3VLclrFN4K!N zDWdL-7Yv7kas9FbQ#WIBDyV`Pjhgow{8&hQyd?Mfk{=n*!ArzNo9o-+?(9X@_I}{@ z+v*b-+4D}WPMxOZ>l6g7&Z2m?ccu+1M=%|CB`YpI43!L%Ke30uiQ%@}{PBzr`na>= zl-|TOW%gM`q+xgKY`28>KwWuzvwf!pv)b=dO>}h1HqG-jC&}C)%cEQre$o1-uhLnf zW*s6qCz4U`<({vV-h7kJ z&m+qu{L+0V7W+&S%0;vbgOZETG}?)?hRa!qZSqgkiD?m9-*1RmUi=E2=Oudw?j_~u ziJ9MqblJ6wzHC?-+W6d-S#G@U;Q_Y%)sa&y=J`MFxCE1c^qaX0w#|;dOdJ!mt)LG|KB-Cje{)DzwPO12vvp7{6INPH2 z=oJ^deF5ZNN-Yj)9uq7bGl}M#;CwJK-p3~S(mbqSAiyi|*6VJgXbz`1@A;aI*EOq& zS^EZ)K?y?dPi~`!&W1Y5Mk|b|csrHoUSWVmP>AB5u20gu-gpAKDZ&9}&k?}qSP5HNe7jT{}xfRVD~_}=1gkv~c9k(!y3`AEP(fDmP^ zYwF}{;|Yc(T^OX(d?B!AhnwuH;$re~&U!m3xrr2`*N+ovNhrkTirYZ+e9fn(LE-t@ zu@(h$KO0>r)S3NmZ7y%>hyISaH|=la+~cFAj<_Y?BJWOlAgd|7jign0K8FXB|KkH~ zDx;#Sk-Le=D5oX)muBUw+!&iiR~1zAAGwp7r1bWY1V>rBbNc>TDa z!d!|hB5iMV%3-;{bv84!?;o@l!b?)lQH!y>4ns(}QzCOMo@I85vOO=T5LR<_^ME@U zIYQaL;jR;>CA8-7(BVxWc(^>Wx~yWOcRC4l{OUf71VKw_N4Wk| zqO%*u1Mz#D4A*6Tmn<9s8jqAn!BYB?=}=m#(x!#_z||f@!6WT9HUR?yo>r@dc*=zd z((p`<3AHypAJ30=s)V1Nojfx)Tf;)v!O=!7S#bPXVvWgixbNbTUVFNPDYnkBa3Em4pjxI0s!O(&5^8!FbvR_nKL|5d9wMO0&z@7K_yj%?uG~5@*_}VDdQp#%u(o1_1eA<0hHC2a8qY)IP#I0sNe|o*t%l9MHyw!MDdT;GR zLgz$^0eW6B?y&CeGj|%jqM&NV_dk#>mzz~A@Th`4bEnujNz$CkOr06sM_Dw2SlDuu zP794pMBp@nZWi5ZXnr8=YWdFT?$RB$BKQ~MD!Y|CKxUOCn2L0JVo(eoI`UCbcm&1NLYuZuP&U{eR;8;Dfm)xRAD|Y`FJ_tdEZr6xT#s1qTX(1s6|SYP zH8R`xjWaY^47}-IMI;p{#%MYBH`j9fjp6Wb^rmp}Wro{_A}+{{6SXf)nqA}2tL8=a z71)&zP_c$t#`)WCofa!uv`ySOpzR;uRJ@q`uC*IQ7SJN7GspBXb>4Bv7t8im;j> z`q4*J*@?)dp_X+WWfrCoR>&%eAnPNZZ)wA5afZlM>^k2C&l#e3qc-(ZjRW$KD06$7 zBlk${q+X{%5L>1;5hIB^k{V$!=#@BZXR`DG&Px@RMb}@5b{bw>rN|m=lkbbv%VG^* zxNEQtdVj>EhlAcK5JbITwfp{)e}cEhpuFLiH>TO#FrALoG8$4^i}DO&WFRuHk=MP} z+-1NZ-&f82XxiHNX$0*hG_2W9j(9PM0*=qVd1iIok+**2|3)8}#7KNS1BvL7e>h@J(s%wUQP`nGi%nGf{)&{Oe}u z{coyYKQvy18I>q`T=Oc0blfnaEpQ>1sl?cErQpx6kM~r58x@YTF`yo=#<;GaaB}+7 za^r|-ylXNJjq4^w6q;q^8``)RA$8;ZVKIYEc;)M*TzikHv+f6637^YXe-xPx6R3iw zWMX@#?j4i$C6E1+w_K*?KQ?mN$zSk$I;{wy!WDRbt*0W2oUEL`_T-T|YtDhW>a&e{ z*k^n@6h>E8p9J@CGwE9=yRXB_(8b`z(;T;pGxTchoQI%md0VHxF7RXaubW7%X6sI& z)N%1Q?`BKy#xxE&+scig=Dih^(VMI5FI(Kq?mdvN5)1pDsVZW#a2z>Dn$K|c)h+8F z(e~XSHn^qUs=M?GW$Nf<<4F%Omo5#xHX*&f+iml+!7e!vhqTeRJTG-nXim1gd(uU6;k zw0CryAlzL!r9BsHs?+7(VMqH@qtux7U30ajM@A8K#yD_K<8zVRKu zVio-`b;|KMA>;|P@%%;+UnSvt3gX={3>e2a8GI2q8rXp_%&@Ov?% z-lV)c@s$s^2scw)N6uuvB=%4|PW>e98+QseHTtAJPfVpz*1vw2XlGZbAfQbdS{z^1@uvjVXhva@cvX-v3Mm4(v}%n_*L(?8UWHJ$&V zZ2n2LX4fLklumhXJRldgV4_&UgigICbtYMNUV44@aAN<>Lb?XaR!Z#Y`-$Sr@w)I@ za1^OtBwS-T>CvwY!K2P{gz`HdeRpD9^kbRjdl{P)xoXSz@YbHqnp*o#sx_OjJHq$k*BHdJaj8?&V{SQvYx^HiF} zy~2~2qX}ve+Hqae=fJJJ(eRBkK~h|k8v1OEgy$Oncr=ph99L;<&7DP}qS+->gHYCZ z>JBd5`~vQBAqQCqbCGaa$*b95!_6p0C6Qnmb013~wV-oL!G7SXXnBFGswRf0<~WIM2D9~Z zWz74$WQRoTo=e#0&wtmwuM!NE&Z=RZ6!(|Wb>0Y>aaHF!KzM_WS9nr4=dcB{BB^@gJhD#CNK@%1$hR&&0>Dd+oB)#jC>7F;Ry zH-X3dnF&UF2`12YU6?|j5Ow{G0}=JiqRB*)L(zt*5+mnIU@v z%X2O4134Gp50{l5K~wpcoH~V2#!oO>OP%4Ah&nl8*Q7TF=fAuyfkw11hBbJ@FG^d zzHdciwL$CJQYS_wb?hu-Di^*U*OXRhc9GwCkB{AQueHc)`Wg<2t_W}@f39$vgs|E4N-T2P(P znv!ZZH@#3`)#lDM)cRw_<&7?ZuFG1wkTYq({NnZq_I`}oRLT9+3%I=!dYM|=63TTNgs@sB5B$N6kgL1xZnxBoLH^$hSXvVdt>dNDc=0D7E{vtb$qm* zPT3jlb^SGl9Ek&VWTauHFj0*!T-a|2^SNb@!lrd?1C zn4JTJd2b^Qaa3wngZH&BwpGRDxBJ_^LLR2t=ajKUm;Y>Dz2{Vtl>4kETRY3u_2tZa z$k{o2!Ogwi?)eNC)-tYf6ts1WoqxtwwK7Y!45nB6{f=#<8mytaM~m7dw|&I0@m?; zTkUX~?56$8_rq)zbER&d)26>Dv75O0Jys;>`g|l*MgHgd`o7GjoWW@Pm#BtAjb|1qVT|!OBJNRV zbzXrGdfGbHmt!MUSxY+lZI z6~CoP3zKqXgCWGlSk`=JCu+({{Ec+{bp?b8{BuNMBgGi2cGLH8$x2+*A23Gvo$a(+ z`38d$nIhcQ;CrnrdGgN_O%_Fsyzp1fH5IY9_6y!81GkW!jvKrZKakFM%R`7XoF9i6;LNTf&>U}ezNuKSznf{E8BVc#^I4M05Ideh&xD*<=}c;3aaQ0s zBXLx*fLn50YaEcl1Z3e)*Hzknk1BT4CMb9=_FU@M1fK!M&+MLwa`Bay)InFYAFd>f z=!^5+KV_(k%-_34p(E3hwzqbBvs!C=7Z>D@X zDxGn?VvO<({4g=oi$8wydSpy1PqXAq1@kiHzR9MJSGs`LzI*;GZ(2#z4pKrxM<3D{Q`&rTLCYd>Bb9cBVbyma{}SM5t%RbDvP?SeW3y;g;0+>DXJcJpYEo^&3^~ z>g~re9M@mA*pPM^a)fa);%UTF&9CUL68TB7Sr=UQMEP3MtEM7b;_89w_@MV$T>bK zK|Qm>YjV_{4>@=bkEIA5fGH3EO`F;hg|7d=uxq+I{~rfIxh{J9 z(Ocw%5FU$DWsS~HK0`}~s1z0ECHPVB`Hm2}9`JY-F`BP89}U-pC<+}|r9$N5`Kc-u zFA5#BHaym?q0B(Ut?^v7xHQkCnb0g%0&B&Wc(E7t-&JtVjWcZp!H0 zD+Ty(+y)L_sNi7Xdt9Z_z6Qz1LHR}blBR|H$x&iRKhhlv-8aD8W$6;)mr#M!eUqXE z!QMYJ_*z~%qtLx^K^XrDk2NRKfn3ieC@qLWmq!rH0RG2!FTj7>3=i$hF*gAKx&>J;pvL+in$t} zK|OOOwCNr1%u&QZft3B`KOOqEIGH{FK(I>RLaJvUs|8nMJ{Qmqzu-)IX;Jq= z;}=?ydGO#W58z{ldDi%zKaT<_g6;#yQUuYZT`b{jMNtcJLA*h?=rn-HHjVj?UarSymc!~UO?fyOg zK_qzL^PeLS>baBizir@d*ivQT7{>YFN4OEa-DVz?^{VRGFv=$+Mx`TuowC6=l>3(}lq55u0a1mWSiznpE zO&0vmzqZ-_pS1U%aL3lE%ntDz)@-cr#8qNeb)AY=G2?W7ot)i_V6O9mt;PZGC}%6k zp&KCAX|0--#PG9(Yw-{}Zdzr#GaUjIP;%HXj3h(MdRW`j}N z@$-aXDa?Hsrj~5dU@5iA{+bk~wMPbQ+Hnc)XE;~rmFIuL703qVtt}Ci4~!u3uYrY4PLMT!%i|fhWrxAsmr>w+)4x<>m?286k7+k_PwP(%n9g zn_HawZN0b`0E50H@!%gg9Jbv)g?9@4OsJ^w7H<){%WF!-FK3o*d8@0Z2BqK*ue*1t zdwQRR&^tN^;g)hYwPgl$!s2`yAS7q+X3bo>X(PFLhNHbECxZ0LJj(n8FP~2@)*SV+ z@$3kq`q;iZ7O(g?H0;Oc17h{hski*hKOCehs{5AaClr1UX3Bd9%5R?q!d`^sge~>w zeXa_O=d&(xJ3!+@wJjNMd0uNVz)M8c-pLXQ+@hflN94!8I2~D;jA176~q^yu=9o zm>X+DTMjRaAgnDw1*M5=_E~6#)8m}V_)?gF2&cTqTZGCtis(CTc@MZLUBf~KTY8Q}iV;mqul(X3Dx&j_K7b#h>Dp6Z$n$-R!5WW_Q|8CvIrw;HDW_O;qPY%&4Ry;_Qs`sIA#~F^> zUcyb)tCfTSeTH^FOQ8%d+0vIrA$7Tj`jJ{jAS{IUndTitkjr+es_eyWe{A>^|5G|I z&u^0P(3+hK&s~5B&%OP8^-2hxRNJhRZQ)$8y*j`++fStjw@B@kb&j9h0k!&@eAI@* zctI=lyDp8(uDFR>KLK}8*oOlg&Y1~UZ z9o3PuVZ$UMg{ai*>EdvD>$N=|>77u(hL(6qI6d?AnqU#M*Fyu4v9Pc>0Spqkej{fBHZ(T%{*`&7klwwP z=lKKg-!EoOo`tEni4h$Ru4q;r2cxlXNWx{2xBu)~*wER=#^y{)L#wsJBC!Qg<6Kg3hQM+e2}uUdwv>~Aiyyys z{39qiKicZd6!o4r7O6|{>RqC1u3s8U3Bm4Q#f|^(Fmk+xfsA!uVVJ zavdfd`@RrOONWx~W^0vKHg0sZ@FK+&)H16k(tU`CT%Mf{+}XxSe;~cxmRxTkdFD?S z8OjHBYI3SAlV);huVI$%d}nGK zXz{t%tmMVys+EL&8=)l!QQZ-f*F*W8~9_ymPNZlZ+qc8`4uen42Y+(?HTZyi%w(w4ciTk+l*-Y-r2?=XvywHfZS zxDkr_@r#H>oyyPow%1&Hh%q8{+l+*Clea3c^R&6->V_)0Ur)>LxxN*dGW?SWrlNdl zinZ$atZ-&JJ!kobqE@dILkPLSECqojfHx<7k0dIN;;2|eMBqNtC3DaIli16J%7D4O z$|YE<4W_TT(-i^?#VIaG=t8WOJ4`;fC24%!h&4vkC#zdTcCMnYmwnGCrhfmlE^!Gd z3ExmxRpj2*EUL^*9u?Cv+a0IAHn5}2ne*e489Qi)dM*F$O*==?$^H8h1A>SROTMpF z8%mQRp-uaH8BvyeOH?sK)IWr#izcPirZ`zBr<*?rJ{**ruy5BGk=a@9gxMD9L@7i4 zuy?rsW=lfc`}3lS5q4Aj8&S^(AyPN9+I>-#VOO&&z7NhAyTjfzILq8pJHjp^WQSdv zHd9Di9$g9k7^!=N_QI#i zKejSFi@eyFz4q~^4BzGY)nGMpnZ3jVEls2X=M250v*2kqKzw?nVMZDR=iaA(LN`d} zUMyyq+P2N%cDK$bP5H*_=hB6&fj7(2g$7quPrDV~bc*9uXvpj^IQQ|kSP$DbDZMU2 z2QYeB2$ zmxQPkAb?w@$i;p;E3|bwuN5(+n_h5e<XE!MWs0*a0up9~($xs!N9% z4?qVKO=Y!MaR)9Zz>+vx?TWWl34UyYvy_UM$`Qmk(BqvqAB=v;YN;Ob4a2$w*o`C~ z6)R%y8b#I=UK*#$=F=!&)?upVmt@T>0k;wjT2LxZdSet6uEg%(9HrNJvtuVJVh*>z z>Gav*53eP-OjMkVBD2Lj*`}JQO)+aR9yVB`Kb9y?E;AcJj|ZHF`C7sk z?6PJiI;Gb=Ymn8#s&_V6aBZ8s;&oWYo~KYOKwx}FBA-#G`iW%Xy#kyw5rQvwaa-g37?N2@ASY zYvariUECn&*IsQq%-6TK*F6VtS-q)hm-K#=_-xULM0Mfg=_8H#4dQAs&h6s*-wBXO zs@-=y05b~W@VMtp`HDv-I7A`YP&j3k6}cMX1%&!-9tO;XfaB5A1azvD)+K+ zHqYt6W5ojckI6nz*Sq6^0=5qDb}hp8v)fKi|DtLiWkULJHD?$usNg5PB&Q-B*c3C( zE=*}!vY30Vtn~QCGe_m;UN-d~cbnUVT}>EezU0psX{#1uiGT?1robKu3}wYVq`XgN zEqvczS%3l;z2uiHmMPZ1cxrhmg%sMnrrQ;G(;c2*h9NwsJi0U{(!y#pkk{{#*046dc}rf_ zbYtuU*&|@LYf4ov}IXW9Lr>D=wcj4rR_XYWl4c}OR_?PsuK z5RwZhR-~Xz@5s+Qc@DZsEta0aHE|CoX7g_!RBRh?|EyQMpzYYgd)cYU?!?Cfiz?Nu zlwA|&^U&ywXT8Bz4h6*v<9Erb0_b8*rer4l9Y{f)>2AuAh|}viKOUUSH*VyAPEdbX z*&)u|)c37SW2r)AJkP=T^LTW0%;l{B`&Y~(2s4$c)9y-(1qGzOH`eF!{Vu)C#?SCa zbjVEh^+OP@&&{+sU`KYi_Tz#`!7B2CTl8t{^@#q}t(Z zXY$Hv-ScXP(++30E;|$obvP78Jy!O~yk(-rWZ=S`q(?*aW-!!UCDL<|H|}hR+iKMx z+xxOrQcXAu7FJOJ{AcU$-B*^e1cSo@fyTXVS)n za=udneo{+iBVb=4ASkCQFp5fr;)#TRGnjK8?lkJY-=@0YLd*X;PE<3nAdLm@dV z+T|6I;~FD&wpUZ+h_9SarrD5hreLCVxp@7peb+Ay1e!s^)DBPKQ^l&x)p~4ZT9*d% zh7KrLEl)B0Lq{Q{BSC9v_(+$8NNoQ3=4%XAxwm6Q&Zgt_K@q_vq8}pjGb=ddWakToNqDQd>c^w8ohSFTfD{ z6T03xAj}I(!=4f`+FJ+*3FF1QNYqr}5s$aDEvS(y7o^_q=UAn;I~b-KqOb$YX&NDd zR9%nq1s)&0uKdjgK`M_!{7c{%E^j4-oMeYrP_-Xk@3+7XNE}p-iMMQ!7-R`x_Ft)W}->C66Y6E8O+)u+v;+|ds3`z@u1w!w?HWdr>T{{aX4Dj9^#gJSw!B)S{7NMnuT1{+1 zkP7IUEg+PmzgSj6zs#!-$f~UVq*JC;sESkwoaGW5Yg^S_BDB>aJK09qk9lKi%F?t$7s^W}_tV}Ue65{oVM7_hnJi`? z75ps(?&b$dPCkUBn_y{*$aEMGh573$FiqIX>4yS74B0dqkOzhw4^o^uQt>Hg*_54r zU5+A&RFJmH2nP0105cwf`SSFleZq>OYf5KG_WNtHb6s_h$nwose*{-{-_#V{xFKD2 zr1l1TH$9P7lMf$rP&7&sHC>n3#H*$qe=~QXd__ujWGc)1_&xA`u~Q8vc_~wKpCL6F zL901rvAge$dtE29Jwvn{Sb-1{o*&ac`Z^MCez$S?u*zpz!p~$>brDXk1$WGccGe(! zy|u8u!dT4gySUIVxOTglTVjT>(dnwUG`k564900?yNJS;hL5Io1);-Q5-iV<3mCpn zh8vmRksjDoBYEI?YH#Mg6L%N84g{|}xsk*dAwbXSP7OFx^F9OhwkN-wD8)wc_cKZda`>f7U1 zuZP{&Z`)Z@dVGhgJ?#GF_#Lioa+w{jZSwytJVF^uCXti)w&jo6_`aH-F*(Y8$BkSNyLy{>K8!Qq5xX;bk|>uR3h2EfWz< z=?&xcf(;2#hPvX-`7T+Sz2!+0r!Z+xebcMebK`|6OkhbM4JnZuO}CY0Y5KDC8Od*cx+^uWFh|$cO0VBzB)YaaJIY4qa+&t9 z0nAoZ$j!mkMt*{OQXC?l11;s=Z*B+)9_s(_sV^n`cIm7*{}bCb93tNEz-9w@%Oi(N ze0GP_TeX}@ph&nj-h8@zOJ_^3GR#1=iAyErki#Wg03L$_84rQFXETuHIbEs`WV zDRG8}Lzk(_9B&Nyb-I!BYIc;{K$*!6&Eh`tT<5|$T(;pA&V2usJ@mmSIb488Ffj<{ z<_`j{2tW9Mbipw2_N=3+s|ou9g#Qle>4BhFav%;2MZ0<8fe=-+Dv|*O`Smo27fSw1>x`@7+6CC z1p9MCHAc8e_ bu7PZViEcr}on&<2+AtlEg2Kr&W}x2y9cN{_ diff --git a/scripts/analyst_collective/__init__.py b/scripts/analyst_collective/__init__.py deleted file mode 100644 index a3b5826..0000000 --- a/scripts/analyst_collective/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ - - -from credentials import Credentials -from runner import Runner diff --git a/scripts/analyst_collective/credentials.py b/scripts/analyst_collective/credentials.py deleted file mode 100644 index a854c78..0000000 --- a/scripts/analyst_collective/credentials.py +++ /dev/null @@ -1,14 +0,0 @@ - - -class Credentials(object): - def __init__(self, filename): - with open(filename) as creds_fh: - creds = creds_fh.read().strip().splitlines() - - if len(creds) != 5: - raise RuntimeError("Credentials file {} invalid!".format(filename)) - - user, pw, host, port, db = creds - - self.conn_string = "dbname='{}' port='{}' user='{}' password='{}' host='{}'".format(db, port, user, pw, host) - diff --git a/scripts/analyst_collective/runner.py b/scripts/analyst_collective/runner.py deleted file mode 100644 index f98362d..0000000 --- a/scripts/analyst_collective/runner.py +++ /dev/null @@ -1,83 +0,0 @@ - -import sqlparse, psycopg2, sys, os, fnmatch - -class Runner(object): - def __init__(self, config, creds, models_dir): - self.config = config - self.creds = creds - self.models_dir = models_dir - - self.connection = psycopg2.connect(creds.conn_string) - - def models(self): - return set(self.config['models']) - - def drop_schema(self): - sql = self.interpolate("drop schema if exists {schema} cascade") - self.execute(sql) - - def create_schema(self): - sql = self.interpolate("create schema {schema}") - self.execute(sql) - - def clean_schema(self): - self.drop_schema() - self.create_schema() - - def execute(self, sql): - debug = sql.replace("\n", " ").strip()[0:200] - print "Running: {}".format(debug) - with self.connection as connection: - with connection.cursor() as cursor: - cursor.execute(sql) - print " {}".format(cursor.statusmessage) - - def interpolate(self, sql, model_name=""): - try: - return sql.format(model=model_name, **self.config) - except KeyError as e: - print "Error interpolating key: {{{error_key}}} in model: {model}".format(error_key=str(e).replace("'", ""), model=model_name) - return "" - - def add_prefix(self, uninterpolated_sql, model): - match = "{schema}." - replace = "{schema}.{model}_" - return uninterpolated_sql.replace(match, replace) - - def __model_files(self): - """returns a dictionary like -{'pardot': ['pardot/model.sql'], - 'segment': ['segment/model.sql'], - 'snowplow': ['snowplow/model.sql'], - 'trello': ['trello/model.sql', 'trello/test.sql']} -""" - indexed_files = {} - for root, dirs, files in os.walk(self.models_dir): - for filename in files: - if fnmatch.fnmatch(filename, "*.sql"): - abs_path = os.path.join(root, filename) - rel_path = os.path.relpath(abs_path, self.models_dir) - namespace = os.path.dirname(rel_path).replace('/', '.') - indexed_files.setdefault(namespace, []).append(rel_path) - return indexed_files - - def create_models(self): - for namespace, files in self.__model_files().iteritems(): - if namespace not in self.models(): - continue - - for f in sorted(files): - model_file = os.path.join(self.models_dir, f) - contents = "" - with open(model_file) as model_fh: - contents = model_fh.read() - - statements = sqlparse.parse(contents) - for statement in statements: - prefixed = self.add_prefix(str(statement), namespace) - sql = self.interpolate(prefixed, namespace) - - if sql is None or len(sql.strip()) == 0: - continue # could throw an error here! Definitely don't execute the sql though - - self.execute(sql) diff --git a/scripts/main.py b/scripts/main.py deleted file mode 100644 index e83e3b1..0000000 --- a/scripts/main.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python - -from analyst_collective import Credentials, Runner -import argparse, json, sys, os - -parser = argparse.ArgumentParser(description='Analyst Collective Runner') -parser.add_argument('--credentials', default="dbcredentials.txt", type=str, help='Path to database credentials file') -parser.add_argument('--config', default="../config.json", type=str, help='Path to analyst collective config file') - -args = parser.parse_args() - -creds = Credentials(args.credentials) - -config = None -with open(args.config) as config_fh: - contents = config_fh.read() - try: - config = json.loads(contents) - except ValueError as e: - print "Could not parse config file {}".format(args.config), e - sys.exit(1) - -models_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'models') - -runner = Runner(config, creds, models_dir) -runner.clean_schema() -runner.create_models() diff --git a/scripts/requirements.txt b/scripts/requirements.txt deleted file mode 100644 index ee9ca6f..0000000 --- a/scripts/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -sqlparse==0.1.18 -wheel==0.24.0 -six==1.10.0 -psycopg2==2.6.1 From 4d9017d8b68644a6f6f456152731d1e9b41384e3 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Thu, 17 Mar 2016 11:41:46 -0400 Subject: [PATCH 22/88] initial stripe model and analysis waiting on test data in redshift to debug this, but looks reasonable for the moment. --- analysis/stripe/analysis.sql | 104 +++++++++++++++++++++++++++++++++++ model/stripe/model.sql | 70 +++++++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 analysis/stripe/analysis.sql create mode 100644 model/stripe/model.sql diff --git a/analysis/stripe/analysis.sql b/analysis/stripe/analysis.sql new file mode 100644 index 0000000..ff912e6 --- /dev/null +++ b/analysis/stripe/analysis.sql @@ -0,0 +1,104 @@ +with + +invoices as ( + select + * + from + {{env.schema}}.stripe_invoices_transformed +), + +plan_changes as ( + select + invoices.company_id as company_id, + invoices.customer as customer, + date_trunc('month', invoices.date) as month, + invoices.total as now, + prior_month_invoices.total as before, + invoices.total - prior_month_invoices.total as change + from + invoices + left outer join + prior_month_invoices + on + date_trunc('month', add_months(invoices.date, -1)) = date_trunc('month', prior_month_invoices.date) + and invoices.customer = prior_month_invoices.customer + where + invoices.forgiven is not true + and invoices.paid is true +), + +news as ( + select + date_trunc('month', date) as month, + sum(total) as total + from + invoices + where + first_payment = 1 + group by + 1 +), + +churns AS ( + select + date_trunc('month', invoices.date) as month, + sum(total) as total + from + invoices + where + last_payment = 1 + and + date_trunc('month', date) < + (select date_trunc('month', current_date - interval '1 month') + ) + group by + 1 +), + +upgrades as ( + select + month, + sum(change) as total + from + plan_changes + where + change > 0 + group by + 1 +), + +downgrades as ( + select + month as month, + sum(change) as total + from + plan_changes + where + change < 0 + group by + 1 +) + +select + upgrades.month, + upgrades.total as upgrades, + coalesce(downgrades.total,0) as downgrades, + coalesce(churns.total,0)*-1 as churn, + coalesce(news.total,0) as news, + case upgrades.month + when (select date_trunc('month', current_date - interval '1 month')) + then 0 + else case coalesce(churns.total,0) + when 0 + then 20 + else coalesce(news.total,0)::float / coalesce(churns.total,0)::float + end + end as quickratio +from upgrades +left outer join downgrades + on upgrades.month = downgrades.month +left outer join churns + on upgrades.month = churns.month +join news + on upgrades.month = news.month +where upgrades.month < (select date_trunc('month', current_date)) diff --git a/model/stripe/model.sql b/model/stripe/model.sql new file mode 100644 index 0000000..70d25a7 --- /dev/null +++ b/model/stripe/model.sql @@ -0,0 +1,70 @@ +create or replace view {{env.schema}}.stripe_invoices as ( + + select + customer, + date, + forgiven, + subscription as subscription_id, + total + from + demo_data.invoices + +); + +create or replace view {{env.schema}}.stripe_subscriptions as ( + + select + id, + plan__id as plan_id + from + demo_data.subscriptions + +); + +create or replace view {{env.schema}}.stripe_plans as ( + + select + production.vero_stripe_production.stripe_plans.id as id, + production.vero_stripe_production.stripe_plans.interval as interval + from + production.vero_stripe_production.stripe_plans + +); + + + + +create or replace view {{env.schema}}.stripe_invoices_transformed as ( + + select + i.customer as customer, + ( + timestamp 'epoch' + i.date * interval '1 Second' + ) as date, + i.forgiven as forgiven, + i.paid as paid, + case p.interval + when 'yearly' + then i.total / 12 / 100 + else i.total / 100 + end as total, + row_number() over( + partition by i.customer + order by i.date desc + ) as last_payment, + row_number() over( + partition by i.customer + order by i.date asc + ) as first_payment + from + {{env.schema}}.stripe_invoices i + join + {{env.schema}}.stripe_subscriptions s + on + i.subscription_id = s.id + join + {{env.schema}}.stripe_plans p + on + s.plan_id = p.id + +); From 2cdc0c801991e1d891025c4a75b5a4343145a59b Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Thu, 17 Mar 2016 12:00:45 -0400 Subject: [PATCH 23/88] minor changes to update plans --- model/stripe/model.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/model/stripe/model.sql b/model/stripe/model.sql index 70d25a7..bb2fe24 100644 --- a/model/stripe/model.sql +++ b/model/stripe/model.sql @@ -7,7 +7,7 @@ create or replace view {{env.schema}}.stripe_invoices as ( subscription as subscription_id, total from - demo_data.invoices + demo_data.stripe_invoices ); @@ -17,17 +17,17 @@ create or replace view {{env.schema}}.stripe_subscriptions as ( id, plan__id as plan_id from - demo_data.subscriptions + demo_data.stripe_subscriptions ); create or replace view {{env.schema}}.stripe_plans as ( select - production.vero_stripe_production.stripe_plans.id as id, - production.vero_stripe_production.stripe_plans.interval as interval + id, + interval from - production.vero_stripe_production.stripe_plans + demo_data.stripe_plans ); @@ -45,7 +45,7 @@ create or replace view {{env.schema}}.stripe_invoices_transformed as ( i.paid as paid, case p.interval when 'yearly' - then i.total / 12 / 100 + then i.total / 12 / 100 else i.total / 100 end as total, row_number() over( From 19563342623e0654e7e1cae2dbd227cbfbe02de5 Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Thu, 17 Mar 2016 16:25:40 -0400 Subject: [PATCH 24/88] zuora v1 --- analysis/mrr/active_mrr.sql | 3 ++- analysis/mrr/total_mrr_by_month.sql | 4 ++-- {models => model}/zuora/model.sql | 22 +++++++++++----------- 3 files changed, 15 insertions(+), 14 deletions(-) rename {models => model}/zuora/model.sql (78%) diff --git a/analysis/mrr/active_mrr.sql b/analysis/mrr/active_mrr.sql index ff5d5a7..b07f11a 100644 --- a/analysis/mrr/active_mrr.sql +++ b/analysis/mrr/active_mrr.sql @@ -1,7 +1,8 @@ +with charges_for_active_plans as ( select * - from ac_yevgeniy.rate_plan_charges + from {{env.schema}}.zuora_subscriptions_w_charges_and_amendments where -- make sure the subscription is active subscr_status = 'Active' diff --git a/analysis/mrr/total_mrr_by_month.sql b/analysis/mrr/total_mrr_by_month.sql index a8fcc7c..896f874 100644 --- a/analysis/mrr/total_mrr_by_month.sql +++ b/analysis/mrr/total_mrr_by_month.sql @@ -7,7 +7,7 @@ dates as from ( select (first_subscr + row_number() over (order by true))::date as date_day - from ac_yevgeniy.rate_plan_charges + from {{env.schema}}.zuora_subscriptions_w_charges_and_amendments ) where date_day <= current_date ), @@ -24,7 +24,7 @@ charges_up_to_each_date as max(date_day) over (partition by subscr_name, dateadd(month,1,date_month)) as max_subscr_trunc_date, max(subscr_version) over (partition by subscr_name, date_month) as max_subscr_version_within_date from dates a - left join ac_yevgeniy.rate_plan_charges b + left join {{env.schema}}.zuora_subscriptions_w_charges_and_amendments b on 1=1 and rpc_start <= date_day and rpc_last_segment = 'TRUE' diff --git a/models/zuora/model.sql b/model/zuora/model.sql similarity index 78% rename from models/zuora/model.sql rename to model/zuora/model.sql index 00ad698..630e968 100644 --- a/models/zuora/model.sql +++ b/model/zuora/model.sql @@ -1,4 +1,4 @@ -create or replace view {{env.schema}}.account as +create or replace view {{env.schema}}.zuora_account as ( select id as account_id, @@ -8,7 +8,7 @@ create or replace view {{env.schema}}.account as ); -create or replace view {{env.schema}}.subscription as +create or replace view {{env.schema}}.zuora_subscription as ( select id as subscr_id, @@ -24,7 +24,7 @@ create or replace view {{env.schema}}.subscription as from zuora.zuora_subscription ); -create or replace view {{env.schema}}.rate_plan as +create or replace view {{env.schema}}.zuora_rate_plan as ( select id as rate_plan_id, @@ -34,7 +34,7 @@ create or replace view {{env.schema}}.rate_plan as ); -create or replace view {{env.schema}}.rate_plan_charge as +create or replace view {{env.schema}}.zuora_rate_plan_charge as ( select rateplanid as rate_plan_id, @@ -47,7 +47,7 @@ create or replace view {{env.schema}}.rate_plan_charge as ); -create or replace view {{env.schema}}.amendment as +create or replace view {{env.schema}}.zuora_amendment as ( select id as amend_id, @@ -59,7 +59,7 @@ create or replace view {{env.schema}}.amendment as -create or replace view {{env.schema}}.rate_plan_charges as +create or replace view {{env.schema}}.zuora_subscriptions_w_charges_and_amendments as ( -- get all subscriptions with possible ammendments for all accounts with subscr_w_amendments as @@ -68,11 +68,11 @@ create or replace view {{env.schema}}.rate_plan_charges as account_number, acc.account_id, sub.subscr_id, subscr_name, subscr_status, subscr_term_type, subscr_start, subscr_end, subscr_version, amend_id, amend_start - from ac_yevgeniy.zuora_account acc - inner join ac_yevgeniy.zuora_subscription sub + from {{env.schema}}.zuora_account acc + inner join {{env.schema}}.zuora_subscription sub on acc.account_id = sub.account_id -- add ammendments - left outer join ac_yevgeniy.zuora_amendment amend + left outer join {{env.schema}}.zuora_amendment amend on sub.subscr_id = amend.subscr_id ) @@ -84,9 +84,9 @@ create or replace view {{env.schema}}.rate_plan_charges as min(subscr_start) over() as first_subscr, "@mrr" as mrr from subscr_w_amendments sub - inner join ac_yevgeniy.zuora_rate_plan rp + inner join {{env.schema}}.zuora_rate_plan rp on rp.subscr_id = sub.subscr_id - inner join ac_yevgeniy.zuora_rate_plan_charge rpc + inner join {{env.schema}}.zuora_rate_plan_charge rpc on rpc.rate_plan_id = rp.rate_plan_id ); From a54b7d3da25c35f380466dd1176eb1e1370a2b45 Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Thu, 17 Mar 2016 16:38:08 -0400 Subject: [PATCH 25/88] zuora v1 --- analysis/mrr/total_mrr_by_month.sql | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/analysis/mrr/total_mrr_by_month.sql b/analysis/mrr/total_mrr_by_month.sql index 896f874..290a50d 100644 --- a/analysis/mrr/total_mrr_by_month.sql +++ b/analysis/mrr/total_mrr_by_month.sql @@ -1,5 +1,11 @@ with +subscriptions as +( + select * + from {{env.schema}}.zuora_subscriptions_w_charges_and_amendments +) + -- genereate calendar dates, starting with the first subscription date dates as ( @@ -7,7 +13,7 @@ dates as from ( select (first_subscr + row_number() over (order by true))::date as date_day - from {{env.schema}}.zuora_subscriptions_w_charges_and_amendments + from subscriptions ) where date_day <= current_date ), @@ -24,7 +30,7 @@ charges_up_to_each_date as max(date_day) over (partition by subscr_name, dateadd(month,1,date_month)) as max_subscr_trunc_date, max(subscr_version) over (partition by subscr_name, date_month) as max_subscr_version_within_date from dates a - left join {{env.schema}}.zuora_subscriptions_w_charges_and_amendments b + left join subscriptions b on 1=1 and rpc_start <= date_day and rpc_last_segment = 'TRUE' From b36f60520fec44a467bb5fcb8a35380b5690026a Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Thu, 17 Mar 2016 17:10:31 -0400 Subject: [PATCH 26/88] zuora v1 --- analysis/mrr/total_mrr_by_month.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analysis/mrr/total_mrr_by_month.sql b/analysis/mrr/total_mrr_by_month.sql index 290a50d..0b618d8 100644 --- a/analysis/mrr/total_mrr_by_month.sql +++ b/analysis/mrr/total_mrr_by_month.sql @@ -4,7 +4,7 @@ subscriptions as ( select * from {{env.schema}}.zuora_subscriptions_w_charges_and_amendments -) +), -- genereate calendar dates, starting with the first subscription date dates as From e4e276997ed28fda386f36be935fc474d5dfb9ef Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Thu, 17 Mar 2016 18:06:08 -0400 Subject: [PATCH 27/88] Merge remote-tracking branch 'origin/master' # Conflicts: # .gitignore --- .gitignore | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.gitignore b/.gitignore index ad2ef72..ea8c4bf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1 @@ - -dbcredentials.txt -env -*.pyc -config.json /target From c9b8c19ef817072e311b71e132c7a2cf33fd0860 Mon Sep 17 00:00:00 2001 From: Matt Monihan Date: Fri, 18 Mar 2016 17:31:20 -0400 Subject: [PATCH 28/88] Magento Hello World. --- analysis/magento/analysis.sql | 42 +++++++++++++++++++++++++++++++++++ model/magento/model.sql | 16 +++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 analysis/magento/analysis.sql create mode 100644 model/magento/model.sql diff --git a/analysis/magento/analysis.sql b/analysis/magento/analysis.sql new file mode 100644 index 0000000..de0e891 --- /dev/null +++ b/analysis/magento/analysis.sql @@ -0,0 +1,42 @@ +/* + +This analysis produces a list of products that includes their: + - sku + - The Product's Name + - Quantity Ordered + - Total Revenue (Price times quantity) + - Total Cost (cost times quantity) + - The Product's cost to the store + - The Product's price to the customer + - The Profit Margin + - The Total Profit + +*/ + +with products as ( + + select * from {{env.schema}}.magento_products + +) + +with order_items as ( + + select * from {{env.schema}}.magento_order_items + +) + +SELECT + products.sku, + products.name, + count(order_items.item_id) as "Quantity", + SUM(order_items.base_price) as "Total Revenue", + products.cost * count(order_items.item_id) as "Total Cost", + products.cost as "Item cost", + base_price as "price", + 1 - (products.cost / base_price) as "profit margin", + SUM(order_items.base_price) - (products.cost * count(order_items.item_id)) as "profit" +FROM {{env.schema}}.magento_order_items +RIGHT JOIN {{env.schema}}.magento_catalog_product_flat_1 as products +ON products.entity_id = order_items.product_id +GROUP BY products.name, order_items.base_price +ORDER BY count(order_items.item_id) desc; diff --git a/model/magento/model.sql b/model/magento/model.sql new file mode 100644 index 0000000..fd2cb08 --- /dev/null +++ b/model/magento/model.sql @@ -0,0 +1,16 @@ +create or replace view {{env.schema}}.magento_order_items as ( + + select * + FROM + magento.sales_flat_order_item + +); + + +create or replace view {{env.schema}}.magento_products as ( + + select * + FROM + magento.catalog_product_flat_1 + +); From a21d445c467c71697ed2a062fef65d306b1e304a Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Sat, 19 Mar 2016 08:41:20 -0400 Subject: [PATCH 29/88] clean up --- README.md | 30 +++++++++++----------------- dbt_project.yml | 16 +++++---------- {model => models}/email/model.sql | 0 {model => models}/pardot/model.sql | 0 {model => models}/pardot/tests.sql | 0 {model => models}/segment/model.sql | 0 {model => models}/snowplow/model.sql | 0 {model => models}/trello/model.sql | 0 {model => models}/trello/test.sql | 0 {model => models}/zuora/model.sql | 0 10 files changed, 17 insertions(+), 29 deletions(-) rename {model => models}/email/model.sql (100%) rename {model => models}/pardot/model.sql (100%) rename {model => models}/pardot/tests.sql (100%) rename {model => models}/segment/model.sql (100%) rename {model => models}/snowplow/model.sql (100%) rename {model => models}/trello/model.sql (100%) rename {model => models}/trello/test.sql (100%) rename {model => models}/zuora/model.sql (100%) diff --git a/README.md b/README.md index 477c7ab..bff1ffd 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,17 @@ ### analyst-collective/models -A collection of model definitions for common data sets in SQL +A collection of data models and corresponding analysis for common data sets in SQL. These models are designed to be portable across organizations with minimal configuration. -### contributing +### Contributing -##### sql construction conventions -- first layer should simply set fields and table / schema -- second layer should be filter. if no records to be filtered out, simply implement as select *. -- third layer should be transformations. this could include datatype conversions, mapping, and other simple transformations to make the data more standardized and consumable. *all transformations must meet the strict definition of universal applicability.* -- see logical-model-flow.pdf for a visual representation of the structure. -- all files should be DDL (should create permanent database objects, not just execute queries) +##### About Models +- A model is a table or view built either on top of raw data or other models. Models are not transient; they are materialized in the database. +- Currently all models are views. Support for models-as-tables is anticipated at some point. +- Model files should go into `/models` and saved with a `.sql` extension. Folder structure within `/models` is for logical grouping only. +- All models are built to be compiled and run with [dbt](https://github.com/analyst-collective/dbt). Environment configuration should be supplied via `{{env}}` +- All models will need to be adapted to your environment so as to select data from the appropriate raw data tables and fields. Once this mapping has been completed, all subsequent analysis built on top of these models will function normally. -### questions -- need to create a destination schema for views created. - - should be separate schema for each source system or all together in a single schema? - - should scripts automatically drop / recreate schemas? much cleaner but high potential for fuckup by users not paying attention. - - should the schema for the intermediate views (base, filtered, transformed) be separate from the schema for the final views? i think so... -- how should unit / integration / regression testing work? the last two, especially, are a huge deal. -- what's the best way to execute a bunch of sql statements in a row even with the sql source being in independent files? -- how do these get documented in an SEO-friendly way? coming across one of these in github will scare off most medium-technical biz users... -- we should consider making a cleanup script that drops all of the views created. it's annoying to clean up after them. -- is there a way we can provide a deduplication layer? this is a big problem that yevgeniy runs into; we should think about it. +##### About Analysis +- All analysis should be built on top of models. +- All named fields in a given analysis should be named within a given model. +- Confining analysis in this way ensures portability of analysis across multiple environments. diff --git a/dbt_project.yml b/dbt_project.yml index acc244a..ea26828 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,12 +1,6 @@ -# Compile configuration -source-paths: ["model"] # paths with source code to compile -target-path: "target" # path for compiled code -clean-targets: ["target"] # directories removed by the clean task - -# Run configuration -# output environments - -run-target: my_redshift - -# Test configuration +#settings specifically for this models directory +#config other dbt settings within ~/.dbt/profiles.yml +source-paths: ["model"] +target-path: "target" +clean-targets: ["target"] test-paths: ["test"] diff --git a/model/email/model.sql b/models/email/model.sql similarity index 100% rename from model/email/model.sql rename to models/email/model.sql diff --git a/model/pardot/model.sql b/models/pardot/model.sql similarity index 100% rename from model/pardot/model.sql rename to models/pardot/model.sql diff --git a/model/pardot/tests.sql b/models/pardot/tests.sql similarity index 100% rename from model/pardot/tests.sql rename to models/pardot/tests.sql diff --git a/model/segment/model.sql b/models/segment/model.sql similarity index 100% rename from model/segment/model.sql rename to models/segment/model.sql diff --git a/model/snowplow/model.sql b/models/snowplow/model.sql similarity index 100% rename from model/snowplow/model.sql rename to models/snowplow/model.sql diff --git a/model/trello/model.sql b/models/trello/model.sql similarity index 100% rename from model/trello/model.sql rename to models/trello/model.sql diff --git a/model/trello/test.sql b/models/trello/test.sql similarity index 100% rename from model/trello/test.sql rename to models/trello/test.sql diff --git a/model/zuora/model.sql b/models/zuora/model.sql similarity index 100% rename from model/zuora/model.sql rename to models/zuora/model.sql From 70dcf7ea2ccd76ef6f92688547bee8306bdbefc3 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Sat, 19 Mar 2016 08:45:23 -0400 Subject: [PATCH 30/88] update dbt project to reference correct models dir --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index ea26828..0eab1f7 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,6 +1,6 @@ #settings specifically for this models directory #config other dbt settings within ~/.dbt/profiles.yml -source-paths: ["model"] +source-paths: ["models"] target-path: "target" clean-targets: ["target"] test-paths: ["test"] From 1046a21fad4f4cc67dc54d644a238fd51deb900b Mon Sep 17 00:00:00 2001 From: Matt Monihan Date: Sun, 20 Mar 2016 10:47:40 -0400 Subject: [PATCH 31/88] Renamed column references and changed schema from 'magento' to 'sample_magento_database'. --- analysis/magento/analysis.sql | 4 ++-- model/magento/model.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/analysis/magento/analysis.sql b/analysis/magento/analysis.sql index de0e891..2c37e29 100644 --- a/analysis/magento/analysis.sql +++ b/analysis/magento/analysis.sql @@ -35,8 +35,8 @@ SELECT base_price as "price", 1 - (products.cost / base_price) as "profit margin", SUM(order_items.base_price) - (products.cost * count(order_items.item_id)) as "profit" -FROM {{env.schema}}.magento_order_items -RIGHT JOIN {{env.schema}}.magento_catalog_product_flat_1 as products +FROM {{env.schema}}.order_items +RIGHT JOIN {{env.schema}}.products ON products.entity_id = order_items.product_id GROUP BY products.name, order_items.base_price ORDER BY count(order_items.item_id) desc; diff --git a/model/magento/model.sql b/model/magento/model.sql index fd2cb08..227e385 100644 --- a/model/magento/model.sql +++ b/model/magento/model.sql @@ -2,7 +2,7 @@ create or replace view {{env.schema}}.magento_order_items as ( select * FROM - magento.sales_flat_order_item + sample_magento_database.sales_flat_order_item ); @@ -11,6 +11,6 @@ create or replace view {{env.schema}}.magento_products as ( select * FROM - magento.catalog_product_flat_1 + sample_magento_database.catalog_product_flat_1 ); From 2b1006b7b94453cdf914af32b810ead0ac679114 Mon Sep 17 00:00:00 2001 From: Matt Monihan Date: Sun, 20 Mar 2016 10:57:44 -0400 Subject: [PATCH 32/88] Adjusted analysis query for magento. Got it running successfully. --- analysis/magento/analysis.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/analysis/magento/analysis.sql b/analysis/magento/analysis.sql index 2c37e29..8226a92 100644 --- a/analysis/magento/analysis.sql +++ b/analysis/magento/analysis.sql @@ -18,8 +18,7 @@ with products as ( select * from {{env.schema}}.magento_products ) - -with order_items as ( +, order_items as ( select * from {{env.schema}}.magento_order_items @@ -35,8 +34,9 @@ SELECT base_price as "price", 1 - (products.cost / base_price) as "profit margin", SUM(order_items.base_price) - (products.cost * count(order_items.item_id)) as "profit" -FROM {{env.schema}}.order_items -RIGHT JOIN {{env.schema}}.products +FROM order_items +RIGHT JOIN products ON products.entity_id = order_items.product_id -GROUP BY products.name, order_items.base_price +WHERE order_items.base_price > 0 +GROUP BY products.sku, products.name, order_items.base_price, products.cost ORDER BY count(order_items.item_id) desc; From c1548c7651c4ea5481114682d2d383f4cf9c1be6 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Mon, 21 Mar 2016 15:26:02 -0400 Subject: [PATCH 33/88] lots of stripe debugging --- analysis/stripe/analysis.sql | 77 ++++++++++++++---------------------- model/stripe/model.sql | 5 ++- 2 files changed, 32 insertions(+), 50 deletions(-) diff --git a/analysis/stripe/analysis.sql b/analysis/stripe/analysis.sql index ff912e6..d33eda0 100644 --- a/analysis/stripe/analysis.sql +++ b/analysis/stripe/analysis.sql @@ -1,15 +1,12 @@ with invoices as ( - select - * - from - {{env.schema}}.stripe_invoices_transformed + select * + from {{env.schema}}.stripe_invoices_transformed ), plan_changes as ( select - invoices.company_id as company_id, invoices.customer as customer, date_trunc('month', invoices.date) as month, invoices.total as now, @@ -17,8 +14,8 @@ plan_changes as ( invoices.total - prior_month_invoices.total as change from invoices - left outer join - prior_month_invoices + left outer join + invoices prior_month_invoices on date_trunc('month', add_months(invoices.date, -1)) = date_trunc('month', prior_month_invoices.date) and invoices.customer = prior_month_invoices.customer @@ -31,52 +28,37 @@ news as ( select date_trunc('month', date) as month, sum(total) as total - from - invoices - where - first_payment = 1 - group by - 1 + from invoices + where first_payment = 1 + group by 1 ), churns AS ( select date_trunc('month', invoices.date) as month, sum(total) as total - from - invoices - where - last_payment = 1 - and - date_trunc('month', date) < - (select date_trunc('month', current_date - interval '1 month') - ) - group by - 1 + from invoices + where last_payment = 1 + and date_trunc('month', date) < date_trunc('month', date_trunc('month', current_date) - 1) + group by 1 ), upgrades as ( select month, sum(change) as total - from - plan_changes - where - change > 0 - group by - 1 + from plan_changes + where change > 0 + group by 1 ), downgrades as ( select month as month, sum(change) as total - from - plan_changes - where - change < 0 - group by - 1 + from plan_changes + where change < 0 + group by 1 ) select @@ -86,19 +68,18 @@ select coalesce(churns.total,0)*-1 as churn, coalesce(news.total,0) as news, case upgrades.month - when (select date_trunc('month', current_date - interval '1 month')) - then 0 - else case coalesce(churns.total,0) - when 0 - then 20 - else coalesce(news.total,0)::float / coalesce(churns.total,0)::float - end + when date_trunc('month', date_trunc('month', current_date) - 1) + then 0 + else + case coalesce(churns.total, 0) + when 0 + then 20 + else + coalesce(news.total, 0)::float / coalesce(churns.total, 0)::float + end end as quickratio from upgrades -left outer join downgrades - on upgrades.month = downgrades.month -left outer join churns - on upgrades.month = churns.month -join news - on upgrades.month = news.month + left outer join downgrades on upgrades.month = downgrades.month + left outer join churns on upgrades.month = churns.month + left outer join news on upgrades.month = news.month where upgrades.month < (select date_trunc('month', current_date)) diff --git a/model/stripe/model.sql b/model/stripe/model.sql index bb2fe24..181af0c 100644 --- a/model/stripe/model.sql +++ b/model/stripe/model.sql @@ -5,6 +5,7 @@ create or replace view {{env.schema}}.stripe_invoices as ( date, forgiven, subscription as subscription_id, + paid, total from demo_data.stripe_invoices @@ -45,8 +46,8 @@ create or replace view {{env.schema}}.stripe_invoices_transformed as ( i.paid as paid, case p.interval when 'yearly' - then i.total / 12 / 100 - else i.total / 100 + then i.total::float / 12 / 100 + else i.total::float / 100 end as total, row_number() over( partition by i.customer From a4b10c157378299815f107794594fdfb28a6a31f Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Tue, 22 Mar 2016 08:30:34 -0400 Subject: [PATCH 34/88] updates to analysis --- analysis/stripe/analysis.sql | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/analysis/stripe/analysis.sql b/analysis/stripe/analysis.sql index d33eda0..4ef4eab 100644 --- a/analysis/stripe/analysis.sql +++ b/analysis/stripe/analysis.sql @@ -61,13 +61,14 @@ downgrades as ( group by 1 ) + select - upgrades.month, + news.month, + coalesce(news.total,0) as news, + coalesce(churns.total,0)*-1 as churn, upgrades.total as upgrades, coalesce(downgrades.total,0) as downgrades, - coalesce(churns.total,0)*-1 as churn, - coalesce(news.total,0) as news, - case upgrades.month + case news.month when date_trunc('month', date_trunc('month', current_date) - 1) then 0 else @@ -78,8 +79,9 @@ select coalesce(news.total, 0)::float / coalesce(churns.total, 0)::float end end as quickratio -from upgrades - left outer join downgrades on upgrades.month = downgrades.month - left outer join churns on upgrades.month = churns.month - left outer join news on upgrades.month = news.month -where upgrades.month < (select date_trunc('month', current_date)) +from news + left outer join upgrades on news.month = upgrades.month + left outer join downgrades on news.month = downgrades.month + left outer join churns on news.month = churns.month +where news.month < (select date_trunc('month', current_date)) +order by month From 55f3600d00a00557be3cb1a1d49c50480a61c764 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Tue, 22 Mar 2016 16:28:39 -0400 Subject: [PATCH 35/88] changes after stripe code review --- analysis/stripe/analysis.sql | 14 ++----- model/stripe/model.sql | 76 ++++++++++++++++++++++-------------- 2 files changed, 49 insertions(+), 41 deletions(-) diff --git a/analysis/stripe/analysis.sql b/analysis/stripe/analysis.sql index 4ef4eab..b611948 100644 --- a/analysis/stripe/analysis.sql +++ b/analysis/stripe/analysis.sql @@ -9,19 +9,11 @@ plan_changes as ( select invoices.customer as customer, date_trunc('month', invoices.date) as month, - invoices.total as now, - prior_month_invoices.total as before, - invoices.total - prior_month_invoices.total as change + lag(invoices.total) over (partition by invoices.customer order by invoices.date) as before, + invoices.total as after, + invoices.total - lag(invoices.total) over (partition by invoices.customer order by invoices.date) as change from invoices - left outer join - invoices prior_month_invoices - on - date_trunc('month', add_months(invoices.date, -1)) = date_trunc('month', prior_month_invoices.date) - and invoices.customer = prior_month_invoices.customer - where - invoices.forgiven is not true - and invoices.paid is true ), news as ( diff --git a/model/stripe/model.sql b/model/stripe/model.sql index 181af0c..76729cb 100644 --- a/model/stripe/model.sql +++ b/model/stripe/model.sql @@ -37,35 +37,51 @@ create or replace view {{env.schema}}.stripe_plans as ( create or replace view {{env.schema}}.stripe_invoices_transformed as ( - select - i.customer as customer, - ( - timestamp 'epoch' + i.date * interval '1 Second' - ) as date, - i.forgiven as forgiven, - i.paid as paid, - case p.interval - when 'yearly' - then i.total::float / 12 / 100 - else i.total::float / 100 - end as total, - row_number() over( - partition by i.customer - order by i.date desc - ) as last_payment, - row_number() over( - partition by i.customer - order by i.date asc - ) as first_payment - from - {{env.schema}}.stripe_invoices i - join - {{env.schema}}.stripe_subscriptions s - on - i.subscription_id = s.id - join - {{env.schema}}.stripe_plans p - on - s.plan_id = p.id + with data as ( + + select + i.customer as customer, + timestamp 'epoch' + i.date * interval '1 Second' as date, + i.forgiven as forgiven, + i.paid as paid, + case p.interval + when 'yearly' + then i.total::float / 12 / 100 + else i.total::float / 100 + end as total, + row_number() over( + partition by i.customer + order by i.date desc + ) as inverse_payment_number, + row_number() over( + partition by i.customer + order by i.date asc + ) as payment_number + from + {{env.schema}}.stripe_invoices i + join + {{env.schema}}.stripe_subscriptions s + on + i.subscription_id = s.id + join + {{env.schema}}.stripe_plans p + on + s.plan_id = p.id + + ) + + select customer, date, total, payment_number, + case inverse_payment_number + when 1 then 1 + else 0 + end as last_payment, + case payment_number + when 1 then 1 + else 0 + end as first_payment + from data + where + forgiven is not true + and paid is true ); From 62fa124f4a2dbc3bdb517d1b689a56ac3e30d897 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Tue, 22 Mar 2016 16:38:39 -0400 Subject: [PATCH 36/88] small updates to analytical query --- analysis/stripe/analysis.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/analysis/stripe/analysis.sql b/analysis/stripe/analysis.sql index b611948..1606bf4 100644 --- a/analysis/stripe/analysis.sql +++ b/analysis/stripe/analysis.sql @@ -2,7 +2,7 @@ with invoices as ( select * - from {{env.schema}}.stripe_invoices_transformed + from ac_jthandy.stripe_invoices_transformed ), plan_changes as ( @@ -31,7 +31,7 @@ churns AS ( sum(total) as total from invoices where last_payment = 1 - and date_trunc('month', date) < date_trunc('month', date_trunc('month', current_date) - 1) + and date_trunc('month', date) < date_trunc('month', current_date) group by 1 ), @@ -75,5 +75,5 @@ from news left outer join upgrades on news.month = upgrades.month left outer join downgrades on news.month = downgrades.month left outer join churns on news.month = churns.month -where news.month < (select date_trunc('month', current_date)) +where news.month < date_trunc('month', current_date) order by month From fc9b2ec5102476f6448f4f60ce942dc4e25f1e46 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Wed, 23 Mar 2016 17:05:42 -0400 Subject: [PATCH 37/88] major refactor of stripe analysis in response to comments in PR --- analysis/stripe/analysis.sql | 158 +++++++++++++++++++++-------------- model/stripe/model.sql | 114 +++++++++++++++---------- 2 files changed, 164 insertions(+), 108 deletions(-) diff --git a/analysis/stripe/analysis.sql b/analysis/stripe/analysis.sql index 1606bf4..8102f1d 100644 --- a/analysis/stripe/analysis.sql +++ b/analysis/stripe/analysis.sql @@ -1,79 +1,109 @@ -with +--bugs: +--for monthly customers who are missing payments, this query records the upgrade when they make the payment but not the prior downgrade when they miss the payment. +--i'm not confident that upgrades and downgrades are having the appropriate amount of revenue called 'renewal' and the appropriate called 'upgrade' or 'downgrade'. +--aka, if a person upgrades from 5 to 50, that's a 5 renewal and a 45 upgrade. from 50 to 5 that's a 45 downgrade and a 5 renewal. +--in general, i would greatly appreciate just some data validation here, making sure that we are extremely confident that the final query here is accurate. + +with invoices as ( -invoices as ( select * - from ac_jthandy.stripe_invoices_transformed -), + from {{env.schema}}.stripe_invoices_transformed + +), all_months as ( + + select distinct date_month from invoices + +), plan_changes as ( -plan_changes as ( select - invoices.customer as customer, - date_trunc('month', invoices.date) as month, - lag(invoices.total) over (partition by invoices.customer order by invoices.date) as before, - invoices.total as after, - invoices.total - lag(invoices.total) over (partition by invoices.customer order by invoices.date) as change + i.*, + pmi.total as prior_month_total, + i.total - coalesce(pmi.total, 0) as change, + pmi.period_end as prior_month_period_end from - invoices -), + invoices i + left outer join invoices pmi + on i.date_month = dateadd('month', 1, pmi.date_month) + and i.customer = pmi.customer -news as ( - select - date_trunc('month', date) as month, - sum(total) as total - from invoices - where first_payment = 1 +), data as ( + + select date_month, total, change, + case + when first_payment = 1 + then 'new' + when last_payment = 1 + and dateadd('month', 1, period_end) < current_date + then 'churn' + when change > 0 + then 'upgrade' + when change < 0 + then 'downgrade' + when period != 'monthly' + and date_month < date_trunc('month', prior_month_period_end) + then 'prepaid renewal' + else + 'renewal' + end revenue_category + from plan_changes + order by customer, payment_number + +), news as ( + + select date_month, sum(total) as value + from data + where revenue_category = 'new' group by 1 -), -churns AS ( - select - date_trunc('month', invoices.date) as month, - sum(total) as total - from invoices - where last_payment = 1 - and date_trunc('month', date) < date_trunc('month', current_date) +), renewals as ( + + select date_month, sum(total) as value + from data + where revenue_category in ('renewal', 'downgrade', 'upgrade') group by 1 -), -upgrades as ( - select - month, - sum(change) as total - from plan_changes - where change > 0 +), prepaids as ( + + select date_month, sum(total) as value + from data + where revenue_category = 'prepaid renewal' group by 1 -), -downgrades as ( - select - month as month, - sum(change) as total - from plan_changes - where change < 0 +), churns as ( + + select date_month, sum(total) as value + from data + where revenue_category = 'churn' group by 1 -) +), upgrades as ( -select - news.month, - coalesce(news.total,0) as news, - coalesce(churns.total,0)*-1 as churn, - upgrades.total as upgrades, - coalesce(downgrades.total,0) as downgrades, - case news.month - when date_trunc('month', date_trunc('month', current_date) - 1) - then 0 - else - case coalesce(churns.total, 0) - when 0 - then 20 - else - coalesce(news.total, 0)::float / coalesce(churns.total, 0)::float - end - end as quickratio -from news - left outer join upgrades on news.month = upgrades.month - left outer join downgrades on news.month = downgrades.month - left outer join churns on news.month = churns.month -where news.month < date_trunc('month', current_date) -order by month + select date_month, sum(change) as value + from data + where revenue_category = 'upgrade' + group by 1 + +), downgrades as ( + + select date_month, sum(change) as value + from data + where revenue_category = 'downgrade' + group by 1 + +) + +select all_months.date_month, + news.value as new, + renewals.value as renewal, + prepaids.value as committed, + churns.value * -1 as churned, + upgrades.value as upgrades, + downgrades.value as downgrades +from all_months + left outer join news on all_months.date_month = news.date_month + left outer join renewals on all_months.date_month = renewals.date_month + left outer join prepaids on all_months.date_month = prepaids.date_month + left outer join churns on all_months.date_month = churns.date_month + left outer join upgrades on all_months.date_month = upgrades.date_month + left outer join downgrades on all_months.date_month = downgrades.date_month +order by 1; diff --git a/model/stripe/model.sql b/model/stripe/model.sql index 76729cb..0a1dabf 100644 --- a/model/stripe/model.sql +++ b/model/stripe/model.sql @@ -6,12 +6,30 @@ create or replace view {{env.schema}}.stripe_invoices as ( forgiven, subscription as subscription_id, paid, - total + total, + period_start, + period_end from demo_data.stripe_invoices ); +create or replace view {{env.schema}}.stripe_invoices_cleaned as ( + + select + customer, + timestamp 'epoch' + date * interval '1 Second' as date, + forgiven, + subscription_id, + paid, + total, + timestamp 'epoch' + period_start * interval '1 Second' as period_start, + timestamp 'epoch' + period_end * interval '1 Second' as period_end + from + {{env.schema}}.stripe_invoices + +); + create or replace view {{env.schema}}.stripe_subscriptions as ( select @@ -37,51 +55,59 @@ create or replace view {{env.schema}}.stripe_plans as ( create or replace view {{env.schema}}.stripe_invoices_transformed as ( - with data as ( - - select - i.customer as customer, - timestamp 'epoch' + i.date * interval '1 Second' as date, - i.forgiven as forgiven, - i.paid as paid, - case p.interval - when 'yearly' - then i.total::float / 12 / 100 - else i.total::float / 100 - end as total, - row_number() over( - partition by i.customer - order by i.date desc - ) as inverse_payment_number, - row_number() over( - partition by i.customer - order by i.date asc - ) as payment_number - from - {{env.schema}}.stripe_invoices i - join - {{env.schema}}.stripe_subscriptions s - on - i.subscription_id = s.id - join - {{env.schema}}.stripe_plans p - on - s.plan_id = p.id + with invoices as ( + + select * + from {{env.schema}}.stripe_invoices_cleaned + + ), d1 as ( + + select (min(period_start) over () + row_number() over ())::date as date_day + from invoices + + ), dates as ( + + select distinct date_trunc('month', date_day)::date as date_month + from d1 + where date_day <= current_date + + ), data as ( + + select date_month, period_start, period_end, + "interval" as period, customer, + case "interval" + when 'yearly' + then i.total::float / 12 / 100 + else i.total::float / 100 + end as total, + row_number() over( + partition by i.customer + order by d.date_month) as payment_number + from dates d + inner join invoices i + on d.date_month::timestamp >= date_trunc('month', i.period_start) + and d.date_month::timestamp < date_trunc('month', i.period_end) + inner join {{env.schema}}.stripe_subscriptions s on i.subscription_id = s.id + inner join {{env.schema}}.stripe_plans p on s.plan_id = p.id + where paid is true + and forgiven is false + order by customer, date_month ) - select customer, date, total, payment_number, - case inverse_payment_number - when 1 then 1 - else 0 - end as last_payment, - case payment_number - when 1 then 1 - else 0 - end as first_payment - from data - where - forgiven is not true - and paid is true + select customer, date_month, total, payment_number, period_end, period, + case + when date_month = last_value(date_month) + over (partition by customer + order by payment_number + rows between unbounded preceding and unbounded following) + then 1 + else 0 + end as last_payment, + case payment_number + when 1 then 1 + else 0 + end as first_payment + from data ); From b66298688a7fed9fba7e5ad5d6aae111e9bcc8bc Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Thu, 24 Mar 2016 00:12:54 -0400 Subject: [PATCH 38/88] more stripe changes --- analysis/stripe/analysis.sql | 21 +++------- model/stripe/model.sql | 81 ++++++++++++++++++++++-------------- 2 files changed, 55 insertions(+), 47 deletions(-) diff --git a/analysis/stripe/analysis.sql b/analysis/stripe/analysis.sql index 8102f1d..8b75125 100644 --- a/analysis/stripe/analysis.sql +++ b/analysis/stripe/analysis.sql @@ -1,9 +1,3 @@ ---bugs: ---for monthly customers who are missing payments, this query records the upgrade when they make the payment but not the prior downgrade when they miss the payment. ---i'm not confident that upgrades and downgrades are having the appropriate amount of revenue called 'renewal' and the appropriate called 'upgrade' or 'downgrade'. ---aka, if a person upgrades from 5 to 50, that's a 5 renewal and a 45 upgrade. from 50 to 5 that's a 45 downgrade and a 5 renewal. ---in general, i would greatly appreciate just some data validation here, making sure that we are extremely confident that the final query here is accurate. - with invoices as ( select * @@ -17,18 +11,14 @@ with invoices as ( select i.*, - pmi.total as prior_month_total, - i.total - coalesce(pmi.total, 0) as change, - pmi.period_end as prior_month_period_end - from - invoices i - left outer join invoices pmi - on i.date_month = dateadd('month', 1, pmi.date_month) - and i.customer = pmi.customer + lag(total) over (partition by customer order by date_month) as prior_month_total, + i.total - coalesce(lag(total) over (partition by customer order by date_month), 0) as change, + lag(period_end) over (partition by customer order by date_month) as prior_month_period_end + from invoices i ), data as ( - select date_month, total, change, + select date_month, total, change case when first_payment = 1 then 'new' @@ -46,7 +36,6 @@ with invoices as ( 'renewal' end revenue_category from plan_changes - order by customer, payment_number ), news as ( diff --git a/model/stripe/model.sql b/model/stripe/model.sql index 0a1dabf..ce2a667 100644 --- a/model/stripe/model.sql +++ b/model/stripe/model.sql @@ -59,6 +59,8 @@ create or replace view {{env.schema}}.stripe_invoices_transformed as ( select * from {{env.schema}}.stripe_invoices_cleaned + where paid is true + and forgiven is false ), d1 as ( @@ -71,43 +73,60 @@ create or replace view {{env.schema}}.stripe_invoices_transformed as ( from d1 where date_day <= current_date + ), customers as ( + + select customer, min(period_start) as active_from, max(period_end) as active_to + from invoices + where period_start <= current_date + group by 1 + + ), customer_dates as ( + + select date_month, customer + from dates d + inner join customers c + on d.date_month >= date_trunc('month', c.active_from) + and d.date_month < date_trunc('month', c.active_to) + ), data as ( - select date_month, period_start, period_end, - "interval" as period, customer, - case "interval" + select date_month, d.customer, period_start, period_end, + "interval" as period, + case "interval" when 'yearly' - then i.total::float / 12 / 100 - else i.total::float / 100 + then coalesce(i.total, 0)::float / 12 / 100 + else + coalesce(i.total, 0)::float / 100 end as total, - row_number() over( - partition by i.customer - order by d.date_month) as payment_number - from dates d - inner join invoices i - on d.date_month::timestamp >= date_trunc('month', i.period_start) - and d.date_month::timestamp < date_trunc('month', i.period_end) - inner join {{env.schema}}.stripe_subscriptions s on i.subscription_id = s.id - inner join {{env.schema}}.stripe_plans p on s.plan_id = p.id - where paid is true - and forgiven is false - order by customer, date_month + first_value(date_month) + over (partition by d.customer + order by date_month + rows between unbounded preceding and unbounded following + ) as first_purchase_month, + last_value(date_month) + over (partition by d.customer + order by date_month + rows between unbounded preceding and unbounded following + ) as last_purchase_month + from customer_dates d + left outer join invoices i + on d.date_month >= date_trunc('month', i.period_start) + and d.date_month < date_trunc('month', i.period_end) + and d.customer = i.customer + left outer join {{env.schema}}.stripe_subscriptions s on i.subscription_id = s.id + left outer join {{env.schema}}.stripe_plans p on s.plan_id = p.id ) - select customer, date_month, total, payment_number, period_end, period, - case - when date_month = last_value(date_month) - over (partition by customer - order by payment_number - rows between unbounded preceding and unbounded following) - then 1 - else 0 - end as last_payment, - case payment_number - when 1 then 1 - else 0 - end as first_payment - from data + select customer, date_month, total, period_end, period, + case first_purchase_month + when date_month then 1 + else 0 + end as first_payment, + case last_purchase_month + when date_month then 1 + else 0 + end as last_payment + from data ); From 8b19da93de08ecce16a619312231170258a96e93 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Thu, 24 Mar 2016 00:48:44 -0400 Subject: [PATCH 39/88] fix renewal calc --- analysis/stripe/analysis.sql | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/analysis/stripe/analysis.sql b/analysis/stripe/analysis.sql index 8b75125..5763c4c 100644 --- a/analysis/stripe/analysis.sql +++ b/analysis/stripe/analysis.sql @@ -18,7 +18,7 @@ with invoices as ( ), data as ( - select date_month, total, change + select *, case when first_payment = 1 then 'new' @@ -34,7 +34,11 @@ with invoices as ( then 'prepaid renewal' else 'renewal' - end revenue_category + end revenue_category, + case + when prior_month_total < total then prior_month_total + else total + end renewal_component_of_change from plan_changes ), news as ( @@ -46,7 +50,7 @@ with invoices as ( ), renewals as ( - select date_month, sum(total) as value + select date_month, sum(renewal_component_of_change) as value from data where revenue_category in ('renewal', 'downgrade', 'upgrade') group by 1 @@ -95,4 +99,4 @@ from all_months left outer join churns on all_months.date_month = churns.date_month left outer join upgrades on all_months.date_month = upgrades.date_month left outer join downgrades on all_months.date_month = downgrades.date_month -order by 1; +order by 1 From 077dd7d248703e90667831641a645ed17faa4f5d Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Thu, 24 Mar 2016 16:40:20 -0400 Subject: [PATCH 40/88] moved magento models to correct folder this was just a mistake before --- {model => models}/magento/model.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {model => models}/magento/model.sql (100%) diff --git a/model/magento/model.sql b/models/magento/model.sql similarity index 100% rename from model/magento/model.sql rename to models/magento/model.sql From e60b8da30be8c98c1dc099476b3992e4b9257658 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Fri, 25 Mar 2016 15:35:17 -0400 Subject: [PATCH 41/88] split out pardot models --- model/pardot/model.sql | 119 ----------------------------------------- 1 file changed, 119 deletions(-) delete mode 100644 model/pardot/model.sql diff --git a/model/pardot/model.sql b/model/pardot/model.sql deleted file mode 100644 index 2f78768..0000000 --- a/model/pardot/model.sql +++ /dev/null @@ -1,119 +0,0 @@ -create or replace view {{env.schema}}.pardot_visitoractivity_types_meta as ( - - --these literal values are pulled from pardot's api docs here: - --http://developer.pardot.com/kb/object-field-references/#visitor-activity - --they change periodically over time and this query will need to be correspondingly modified. - select 1 as type, 'Click' as type_decoded union all - select 2, 'View' union all - select 3, 'Error' union all - select 4, 'Success' union all - select 5, 'Session' union all - select 6, 'Sent' union all - select 7, 'Search' union all - select 8, 'New Opportunity' union all - select 9, 'Opportunity Won' union all - select 10, 'Opportunity Lost' union all - select 11, 'Open' union all - select 12, 'Unsubscribe Page' union all - select 13, 'Bounced' union all - select 14, 'Spam Complaint' union all - select 15, 'Email Preference Page' union all - select 16, 'Resubscribed' union all - select 17, 'Click (Third Party)' union all - select 18, 'Opportunity Reopened' union all - select 19, 'Opportunity Linked' union all - select 20, 'Visit' union all - select 21, 'Custom URL click' union all - select 22, 'Olark Chat' union all - select 23, 'Invited to Webinar' union all - select 24, 'Attended Webinar' union all - select 25, 'Registered for Webinar' union all - select 26, 'Social Post Click' union all - select 27, 'Video View' union all - select 28, 'Event Registered' union all - select 29, 'Event Checked In' union all - select 30, 'Video Conversion' union all - select 31, 'UserVoice Suggestion' union all - select 32, 'UserVoice Comment' union all - select 33, 'UserVoice Ticket' union all - select 34, 'Video Watched (>= 75% watched)' - -); - - - -create or replace view {{env.schema}}.pardot_visitoractivity_events_meta as ( - - --even with the type decoding that Pardot specifically provides, actually what is going on in a given event - --is somewhat ambiguous. this is an attempt to map type and type_name to a more event-based "event action" field - --which is always written in more standard action-oriented terms. - select 22 as "type", 'Chat Transcript' as type_name, 'chatted via olark' as event_name union all - select 21, 'Custom Redirect', 'clicked a custom redirect' union all - select 6, 'Email', 'email sent' union all - select 11, 'Email', 'email opened' union all - select 13, 'Email', 'email bounced' union all - select 14, 'Email', 'email reported spam' union all - select 1, 'Email Tracker', 'email click' union all - select 28, 'Event', 'registered for event' union all - select 29, 'Event', 'checked in at event' union all - select 2, 'File', 'viewed a file' union all - select 3, 'Form', 'submitted a form with an error' union all - select 2, 'Form', 'viewed a form' union all - select 4, 'Form', 'successfully submitted a form' union all - select 4, 'Form Handler', 'successfully submitted a form handler' union all - select 2, 'Landing Page', 'viewed a landing page' union all - select 4, 'Landing Page', 'successfully submitted the form on a landing page' union all - select 3, 'Landing Page', 'submitted the form on a landing page with an error' union all - select 2, 'Multivariate Landing Page', 'viewed multivariate landing page' union all - select 4, 'Multivariate Landing Page', 'successfully submitted multivariate landing page' union all - select 3, 'Multivariate Landing Page', 'submitted multivariate landing page with an error' union all - select 8, 'New Opportunity', 'opened opportunity' union all - select 19, 'Opportunity Associated', 'linked existing opportunity' union all - select 10, 'Opportunity Lost', 'lost opportunity' union all - select 9, 'Opportunity Won', 'won opportunity' union all - select 2, 'Page View', 'viewed highlighted page' union all - select 34, 'Video', 'watched 75% or more of video' union all - select 27, 'Video', 'watched video' union all - select 30, 'Video', 'converted from video call to action' union all - select 20, 'Visit', 'visited website' union all - select 25, 'Webinar', 'registered for webinar' union all - select 24, 'Webinar', 'attended webinar' union all - select 18, '', 'reopened opportunity' - -); - - -create or replace view {{env.schema}}.pardot_visitoractivity as ( - --this table has a bunch of types that really should be event actions but are very poorly formulated. - --the custom logic in this view is an attempt to fix that. - --not all of the various type / type_name combinations have been accounted for yet; I still need to determine exactly what some of them mean. - select - -- event_stream interface - va.created_at as "@timestamp", - e.event_name as "@event", - va.prospect_id as "@user_id", - va.* - from - olga_pardot.visitoractivity va - inner join {{env.schema}}.pardot_visitoractivity_events_meta e - on va."type" = e."type" and va.type_name = e.type_name - inner join {{env.schema}}.pardot_visitoractivity_types_meta t - on va."type" = t."type" -); - -COMMENT ON VIEW {{env.schema}}.pardot_visitoractivity IS 'timeseries,funnel,cohort'; - - -/* -This model maps pardot data from the visitoractivity table to the email analysis interface. -It conforms to the basic email interface, not the extended email interface, because Pardot does not supply -data necessary to conform to the extended interface. -*/ - -create or replace view {{env.schema}}.emails as ( - - select "@timestamp", "@event", "@user_id", email_id as "@email_id", details as "@subject" - from {{env.schema}}.pardot_visitoractivity - where "@event" in ('email sent', 'email opened', 'email click') - -); From 5ee44dffc0a7e0f391c59076e583d3a0a05c3dab Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Fri, 25 Mar 2016 16:17:51 -0400 Subject: [PATCH 42/88] add new pardot files --- model/pardot/emails.sql | 9 +++++ model/pardot/visitoractivity.sql | 15 ++++++++ model/pardot/visitoractivity_events_meta.sql | 35 ++++++++++++++++++ model/pardot/visitoractivity_types_meta.sql | 37 ++++++++++++++++++++ 4 files changed, 96 insertions(+) create mode 100644 model/pardot/emails.sql create mode 100644 model/pardot/visitoractivity.sql create mode 100644 model/pardot/visitoractivity_events_meta.sql create mode 100644 model/pardot/visitoractivity_types_meta.sql diff --git a/model/pardot/emails.sql b/model/pardot/emails.sql new file mode 100644 index 0000000..b8308f1 --- /dev/null +++ b/model/pardot/emails.sql @@ -0,0 +1,9 @@ +/* +This model maps pardot data from the visitoractivity table to the email analysis interface. +It conforms to the basic email interface, not the extended email interface, because Pardot does not supply +data necessary to conform to the extended interface. +*/ + +select "@timestamp", "@event", "@user_id", email_id as "@email_id", details as "@subject" + from {{env.schema}}.pardot_visitoractivity +where "@event" in ('email sent', 'email opened', 'email click') diff --git a/model/pardot/visitoractivity.sql b/model/pardot/visitoractivity.sql new file mode 100644 index 0000000..dceb6dc --- /dev/null +++ b/model/pardot/visitoractivity.sql @@ -0,0 +1,15 @@ +--this table has a bunch of types that really should be event actions but are very poorly formulated. +--the custom logic in this view is an attempt to fix that. +--not all of the various type / type_name combinations have been accounted for yet; I still need to determine exactly what some of them mean. +select + -- event_stream interface + va.created_at as "@timestamp", + e.event_name as "@event", + va.prospect_id as "@user_id", + va.* +from + olga_pardot.visitoractivity va + inner join {{env.schema}}.visitoractivity_events_meta e + on va."type" = e."type" and va.type_name = e.type_name + inner join {{env.schema}}.visitoractivity_types_meta t + on va."type" = t."type" diff --git a/model/pardot/visitoractivity_events_meta.sql b/model/pardot/visitoractivity_events_meta.sql new file mode 100644 index 0000000..2339b65 --- /dev/null +++ b/model/pardot/visitoractivity_events_meta.sql @@ -0,0 +1,35 @@ +--even with the type decoding that Pardot specifically provides, actually what is going on in a given event +--is somewhat ambiguous. this is an attempt to map type and type_name to a more event-based "event action" field +--which is always written in more standard action-oriented terms. +select 22 as "type", 'Chat Transcript' as type_name, 'chatted via olark' as event_name union all +select 21, 'Custom Redirect', 'clicked a custom redirect' union all +select 6, 'Email', 'email sent' union all +select 11, 'Email', 'email opened' union all +select 13, 'Email', 'email bounced' union all +select 14, 'Email', 'email reported spam' union all +select 1, 'Email Tracker', 'email click' union all +select 28, 'Event', 'registered for event' union all +select 29, 'Event', 'checked in at event' union all +select 2, 'File', 'viewed a file' union all +select 3, 'Form', 'submitted a form with an error' union all +select 2, 'Form', 'viewed a form' union all +select 4, 'Form', 'successfully submitted a form' union all +select 4, 'Form Handler', 'successfully submitted a form handler' union all +select 2, 'Landing Page', 'viewed a landing page' union all +select 4, 'Landing Page', 'successfully submitted the form on a landing page' union all +select 3, 'Landing Page', 'submitted the form on a landing page with an error' union all +select 2, 'Multivariate Landing Page', 'viewed multivariate landing page' union all +select 4, 'Multivariate Landing Page', 'successfully submitted multivariate landing page' union all +select 3, 'Multivariate Landing Page', 'submitted multivariate landing page with an error' union all +select 8, 'New Opportunity', 'opened opportunity' union all +select 19, 'Opportunity Associated', 'linked existing opportunity' union all +select 10, 'Opportunity Lost', 'lost opportunity' union all +select 9, 'Opportunity Won', 'won opportunity' union all +select 2, 'Page View', 'viewed highlighted page' union all +select 34, 'Video', 'watched 75% or more of video' union all +select 27, 'Video', 'watched video' union all +select 30, 'Video', 'converted from video call to action' union all +select 20, 'Visit', 'visited website' union all +select 25, 'Webinar', 'registered for webinar' union all +select 24, 'Webinar', 'attended webinar' union all +select 18, '', 'reopened opportunity' diff --git a/model/pardot/visitoractivity_types_meta.sql b/model/pardot/visitoractivity_types_meta.sql new file mode 100644 index 0000000..0e664bf --- /dev/null +++ b/model/pardot/visitoractivity_types_meta.sql @@ -0,0 +1,37 @@ +--these literal values are pulled from pardot's api docs here: +--http://developer.pardot.com/kb/object-field-references/#visitor-activity +--they change periodically over time and this query will need to be correspondingly modified. +select 1 as type, 'Click' as type_decoded union all +select 2, 'View' union all +select 3, 'Error' union all +select 4, 'Success' union all +select 5, 'Session' union all +select 6, 'Sent' union all +select 7, 'Search' union all +select 8, 'New Opportunity' union all +select 9, 'Opportunity Won' union all +select 10, 'Opportunity Lost' union all +select 11, 'Open' union all +select 12, 'Unsubscribe Page' union all +select 13, 'Bounced' union all +select 14, 'Spam Complaint' union all +select 15, 'Email Preference Page' union all +select 16, 'Resubscribed' union all +select 17, 'Click (Third Party)' union all +select 18, 'Opportunity Reopened' union all +select 19, 'Opportunity Linked' union all +select 20, 'Visit' union all +select 21, 'Custom URL click' union all +select 22, 'Olark Chat' union all +select 23, 'Invited to Webinar' union all +select 24, 'Attended Webinar' union all +select 25, 'Registered for Webinar' union all +select 26, 'Social Post Click' union all +select 27, 'Video View' union all +select 28, 'Event Registered' union all +select 29, 'Event Checked In' union all +select 30, 'Video Conversion' union all +select 31, 'UserVoice Suggestion' union all +select 32, 'UserVoice Comment' union all +select 33, 'UserVoice Ticket' union all +select 34, 'Video Watched (>= 75% watched)' From 847bb21002644570ad952f897c4501d050d67682 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Fri, 25 Mar 2016 16:40:25 -0400 Subject: [PATCH 43/88] fix pardot defs and split out magento models --- dbt_project.yml | 3 +++ model/magento/model.sql | 16 ------------ model/magento/order_items.sql | 3 +++ model/magento/products.sql | 4 +++ .../pardot/{emails.sql => pardot_emails.sql} | 0 model/pardot/pardot_tests.sql | 20 ++++++++++++++ ...ctivity.sql => pardot_visitoractivity.sql} | 4 +-- ...=> pardot_visitoractivity_events_meta.sql} | 0 ... => pardot_visitoractivity_types_meta.sql} | 0 model/pardot/tests.sql | 26 ------------------- 10 files changed, 32 insertions(+), 44 deletions(-) delete mode 100644 model/magento/model.sql create mode 100644 model/magento/order_items.sql create mode 100644 model/magento/products.sql rename model/pardot/{emails.sql => pardot_emails.sql} (100%) create mode 100644 model/pardot/pardot_tests.sql rename model/pardot/{visitoractivity.sql => pardot_visitoractivity.sql} (81%) rename model/pardot/{visitoractivity_events_meta.sql => pardot_visitoractivity_events_meta.sql} (100%) rename model/pardot/{visitoractivity_types_meta.sql => pardot_visitoractivity_types_meta.sql} (100%) delete mode 100644 model/pardot/tests.sql diff --git a/dbt_project.yml b/dbt_project.yml index acc244a..78eefc0 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -6,6 +6,9 @@ clean-targets: ["target"] # directories removed by the clean task # Run configuration # output environments +models: ['pardot'] +table_or_view: view + run-target: my_redshift # Test configuration diff --git a/model/magento/model.sql b/model/magento/model.sql deleted file mode 100644 index 227e385..0000000 --- a/model/magento/model.sql +++ /dev/null @@ -1,16 +0,0 @@ -create or replace view {{env.schema}}.magento_order_items as ( - - select * - FROM - sample_magento_database.sales_flat_order_item - -); - - -create or replace view {{env.schema}}.magento_products as ( - - select * - FROM - sample_magento_database.catalog_product_flat_1 - -); diff --git a/model/magento/order_items.sql b/model/magento/order_items.sql new file mode 100644 index 0000000..07d33d3 --- /dev/null +++ b/model/magento/order_items.sql @@ -0,0 +1,3 @@ +select * +FROM + sample_magento_database.sales_flat_order_item diff --git a/model/magento/products.sql b/model/magento/products.sql new file mode 100644 index 0000000..87468fb --- /dev/null +++ b/model/magento/products.sql @@ -0,0 +1,4 @@ +select * +FROM + sample_magento_database.catalog_product_flat_1 + diff --git a/model/pardot/emails.sql b/model/pardot/pardot_emails.sql similarity index 100% rename from model/pardot/emails.sql rename to model/pardot/pardot_emails.sql diff --git a/model/pardot/pardot_tests.sql b/model/pardot/pardot_tests.sql new file mode 100644 index 0000000..20155df --- /dev/null +++ b/model/pardot/pardot_tests.sql @@ -0,0 +1,20 @@ +select + 'visitoractivity_fresher_than_one_day' as name, + 'Most recent visitoractivity entry is no more than one day old' as description, + max("@timestamp"::timestamp) > current_date - '1 day'::interval as result +from {{env.schema}}.pardot_visitoractivity + + + + + + + +/* + +Other tests I want to do: +- make sure there are records from every day since the first day we see any records +- make sure all prospect ids from visitoractivity show up in prospects +- make sure there are no unmapped types + +*/ diff --git a/model/pardot/visitoractivity.sql b/model/pardot/pardot_visitoractivity.sql similarity index 81% rename from model/pardot/visitoractivity.sql rename to model/pardot/pardot_visitoractivity.sql index dceb6dc..f56509a 100644 --- a/model/pardot/visitoractivity.sql +++ b/model/pardot/pardot_visitoractivity.sql @@ -9,7 +9,7 @@ select va.* from olga_pardot.visitoractivity va - inner join {{env.schema}}.visitoractivity_events_meta e + inner join {{env.schema}}.pardot_visitoractivity_events_meta e on va."type" = e."type" and va.type_name = e.type_name - inner join {{env.schema}}.visitoractivity_types_meta t + inner join {{env.schema}}.pardot_visitoractivity_types_meta t on va."type" = t."type" diff --git a/model/pardot/visitoractivity_events_meta.sql b/model/pardot/pardot_visitoractivity_events_meta.sql similarity index 100% rename from model/pardot/visitoractivity_events_meta.sql rename to model/pardot/pardot_visitoractivity_events_meta.sql diff --git a/model/pardot/visitoractivity_types_meta.sql b/model/pardot/pardot_visitoractivity_types_meta.sql similarity index 100% rename from model/pardot/visitoractivity_types_meta.sql rename to model/pardot/pardot_visitoractivity_types_meta.sql diff --git a/model/pardot/tests.sql b/model/pardot/tests.sql deleted file mode 100644 index 092c605..0000000 --- a/model/pardot/tests.sql +++ /dev/null @@ -1,26 +0,0 @@ -create or replace view {{env.schema}}.pardot_model_tests - (name, description, result) - as ( - - select - 'visitoractivity_fresher_than_one_day', - 'Most recent visitoractivity entry is no more than one day old', - max("@timestamp"::timestamp) > current_date - '1 day'::interval - from {{env.schema}}.pardot_visitoractivity - - ); - - - - - - - - /* - -Other tests I want to do: - - make sure there are records from every day since the first day we see any records - - make sure all prospect ids from visitoractivity show up in prospects - - make sure there are no unmapped types - - */ From 4499b0db8c0fd693b8ddb416b9c0ec8172423871 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Fri, 25 Mar 2016 16:41:35 -0400 Subject: [PATCH 44/88] split out emails --- model/email/emails_denormalized.sql | 36 ++++++++++++++++++++++++++ model/email/model.sql | 40 ----------------------------- 2 files changed, 36 insertions(+), 40 deletions(-) create mode 100644 model/email/emails_denormalized.sql delete mode 100644 model/email/model.sql diff --git a/model/email/emails_denormalized.sql b/model/email/emails_denormalized.sql new file mode 100644 index 0000000..7acc8f1 --- /dev/null +++ b/model/email/emails_denormalized.sql @@ -0,0 +1,36 @@ +with events as ( + + select * from {{env.schema}}.emails + +), + +sends as ( + + select "@user_id", "@timestamp", "@email_id" + from events + where "@event" = 'email sent' + +), + +opens as ( + + select "@user_id", "@timestamp", "@email_id" + from events + where "@event" = 'email opened' + +), clicks as ( + + select "@user_id", "@timestamp", "@email_id" + from events + where "@event" = 'email click' + +) + +select s."@user_id", s."@timestamp" as sent_timestamp, o."@timestamp" as opened_timestamp, c."@timestamp" as clicked_timestamp, + decode(o."@timestamp", null, 0, 1) as "opened?", + decode(c."@timestamp", null, 0, 1) as "clicked?", + row_number() over (partition by s."@user_id" order by s."@timestamp") as email_number +from sends s + left outer join opens o on s."@email_id" = o."@email_id" + left outer join clicks c on s."@email_id" = c."@email_id" +order by 1, 2 diff --git a/model/email/model.sql b/model/email/model.sql deleted file mode 100644 index 0365b95..0000000 --- a/model/email/model.sql +++ /dev/null @@ -1,40 +0,0 @@ -create or replace view {{env.schema}}.emails_denormalized as ( - - with events as ( - - select * from {{env.schema}}.emails - - ), - - sends as ( - - select "@user_id", "@timestamp", "@email_id" - from events - where "@event" = 'email sent' - - ), - - opens as ( - - select "@user_id", "@timestamp", "@email_id" - from events - where "@event" = 'email opened' - - ), clicks as ( - - select "@user_id", "@timestamp", "@email_id" - from events - where "@event" = 'email click' - - ) - - select s."@user_id", s."@timestamp" as sent_timestamp, o."@timestamp" as opened_timestamp, c."@timestamp" as clicked_timestamp, - decode(o."@timestamp", null, 0, 1) as "opened?", - decode(c."@timestamp", null, 0, 1) as "clicked?", - row_number() over (partition by s."@user_id" order by s."@timestamp") as email_number - from sends s - left outer join opens o on s."@email_id" = o."@email_id" - left outer join clicks c on s."@email_id" = c."@email_id" - order by 1, 2 - -); From 39f4ad84a299c0d66d6e4a2e9eab93977464f9ac Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Fri, 25 Mar 2016 16:42:40 -0400 Subject: [PATCH 45/88] split out snowplow and segment --- model/segment/model.sql | 12 ------------ model/segment/segment_track.sql | 8 ++++++++ model/snowplow/model.sql | 11 ----------- model/snowplow/snowplow_events.sql | 7 +++++++ 4 files changed, 15 insertions(+), 23 deletions(-) delete mode 100644 model/segment/model.sql create mode 100644 model/segment/segment_track.sql delete mode 100644 model/snowplow/model.sql create mode 100644 model/snowplow/snowplow_events.sql diff --git a/model/segment/model.sql b/model/segment/model.sql deleted file mode 100644 index 7a10851..0000000 --- a/model/segment/model.sql +++ /dev/null @@ -1,12 +0,0 @@ -create or replace view {{env.schema}}.segment_track as ( - select - "timestamp"::timestamp as "@timestamp", - "event" as "@event", - "userid" as "@user_id", - * - - from - segment.track -); - -comment on view {{env.schema}}.segment_track is 'timeseries,funnel,cohort'; diff --git a/model/segment/segment_track.sql b/model/segment/segment_track.sql new file mode 100644 index 0000000..b393fea --- /dev/null +++ b/model/segment/segment_track.sql @@ -0,0 +1,8 @@ +select + "timestamp"::timestamp as "@timestamp", + "event" as "@event", + "userid" as "@user_id", + * + +from + segment.track diff --git a/model/snowplow/model.sql b/model/snowplow/model.sql deleted file mode 100644 index e81a581..0000000 --- a/model/snowplow/model.sql +++ /dev/null @@ -1,11 +0,0 @@ -create or replace view {{env.schema}}.snowplow_events as ( - select - "collector_tstamp" as "@timestamp", - "event_name" as "@event", - "domain_userid" as "@user_id", - * - from - atomic.events -); - -comment on view {{env.schema}}.snowplow_events is 'timeseries,funnel,cohort'; diff --git a/model/snowplow/snowplow_events.sql b/model/snowplow/snowplow_events.sql new file mode 100644 index 0000000..f438b35 --- /dev/null +++ b/model/snowplow/snowplow_events.sql @@ -0,0 +1,7 @@ +select + "collector_tstamp" as "@timestamp", + "event_name" as "@event", + "domain_userid" as "@user_id", + * +from + atomic.events From 8c8aad26f914773b5e642dd41582f01d5136658c Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Fri, 25 Mar 2016 16:43:15 -0400 Subject: [PATCH 46/88] split out trello model --- model/trello/model.sql | 20 -------------------- model/trello/trello_card_location.sql | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 20 deletions(-) delete mode 100644 model/trello/model.sql create mode 100644 model/trello/trello_card_location.sql diff --git a/model/trello/model.sql b/model/trello/model.sql deleted file mode 100644 index 3eee29f..0000000 --- a/model/trello/model.sql +++ /dev/null @@ -1,20 +0,0 @@ -create or replace view {{env.schema}}.trello_card_location as ( - select - id, - idmembercreator, - date, - "type", - data__card__id, - data__card__name, - coalesce(data__list__id, - data__listafter__id, - lag(coalesce(data__list__id, data__listafter__id)) ignore nulls over (partition by data__card__id order by date)) as data__list__id, - coalesce(data__boardtarget__id, data__board__id) as data__board__id, - coalesce(data__card__closed, - lag(data__card__closed) ignore nulls over (partition by data__card__id order by date), - false) as data__card__closed - from trello_growth.trello_actions - where - data__card__id is not null - and "type" in ('createCard', 'updateCard', 'moveCardFromBoard', 'moveCardToBoard', 'commentCard') -); diff --git a/model/trello/trello_card_location.sql b/model/trello/trello_card_location.sql new file mode 100644 index 0000000..f6ebe45 --- /dev/null +++ b/model/trello/trello_card_location.sql @@ -0,0 +1,18 @@ +select + id, + idmembercreator, + date, + "type", + data__card__id, + data__card__name, + coalesce(data__list__id, + data__listafter__id, + lag(coalesce(data__list__id, data__listafter__id)) ignore nulls over (partition by data__card__id order by date)) as data__list__id, + coalesce(data__boardtarget__id, data__board__id) as data__board__id, + coalesce(data__card__closed, + lag(data__card__closed) ignore nulls over (partition by data__card__id order by date), + false) as data__card__closed +from trello_growth.trello_actions +where + data__card__id is not null + and "type" in ('createCard', 'updateCard', 'moveCardFromBoard', 'moveCardToBoard', 'commentCard') From 3caf061a879c029591d856e1d191dc78b436b333 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Fri, 25 Mar 2016 16:44:18 -0400 Subject: [PATCH 47/88] restructure trello tests --- model/trello/test.sql | 25 ------------------------- model/trello/trello_model_tests.sql | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 25 deletions(-) delete mode 100644 model/trello/test.sql create mode 100644 model/trello/trello_model_tests.sql diff --git a/model/trello/test.sql b/model/trello/test.sql deleted file mode 100644 index f2d5a9b..0000000 --- a/model/trello/test.sql +++ /dev/null @@ -1,25 +0,0 @@ -create or replace view {{env.schema}}.trello_model_tests -(name, description, result) -as ( - with null_boards_or_lists as - ( - select id - from {{env.schema}}.trello_card_location - where - data__board__id is null - or data__list__id is null - ) - select - 'no_null_boards_or_lists', - 'All location entries have a non-null board and list id', - count(*) = 0 - from null_boards_or_lists - - union all - - select - 'fresher_than_one_day', - 'Most recent entry is no more than one day old', - max(date::timestamp) > current_date - '1 day'::interval - from {{env.schema}}.trello_card_location -); diff --git a/model/trello/trello_model_tests.sql b/model/trello/trello_model_tests.sql new file mode 100644 index 0000000..9f9840b --- /dev/null +++ b/model/trello/trello_model_tests.sql @@ -0,0 +1,21 @@ +with null_boards_or_lists as +( + select id + from {{env.schema}}.trello_card_location + where + data__board__id is null + or data__list__id is null +) +select + 'no_null_boards_or_lists' as name, + 'All location entries have a non-null board and list id' as description, +count(*) = 0 as result +from null_boards_or_lists + +union all + +select + 'fresher_than_one_day', + 'Most recent entry is no more than one day old', +max(date::timestamp) > current_date - '1 day'::interval +from {{env.schema}}.trello_card_location From a43a8a4a802564ae0198bb4323d7c7025d147725 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Fri, 25 Mar 2016 16:49:52 -0400 Subject: [PATCH 48/88] split out zuora models --- model/zuora/model.sql | 95 ------------------- model/zuora/zuora_account.sql | 5 + model/zuora/zuora_amendment.sql | 6 ++ model/zuora/zuora_rate_plan.sql | 5 + model/zuora/zuora_rate_plan_charge.sql | 8 ++ model/zuora/zuora_subscription.sql | 12 +++ ...subscriptions_w_charges_and_amendments.sql | 27 ++++++ 7 files changed, 63 insertions(+), 95 deletions(-) delete mode 100644 model/zuora/model.sql create mode 100644 model/zuora/zuora_account.sql create mode 100644 model/zuora/zuora_amendment.sql create mode 100644 model/zuora/zuora_rate_plan.sql create mode 100644 model/zuora/zuora_rate_plan_charge.sql create mode 100644 model/zuora/zuora_subscription.sql create mode 100644 model/zuora/zuora_subscriptions_w_charges_and_amendments.sql diff --git a/model/zuora/model.sql b/model/zuora/model.sql deleted file mode 100644 index 630e968..0000000 --- a/model/zuora/model.sql +++ /dev/null @@ -1,95 +0,0 @@ -create or replace view {{env.schema}}.zuora_account as -( - select - id as account_id, - accountnumber as account_number, - * - from zuora.zuora_account -); - - -create or replace view {{env.schema}}.zuora_subscription as -( - select - id as subscr_id, - status as subscr_status, - termtype as subscr_term_type, - accountid as account_id, - contracteffectivedate::timestamp as subscr_start, - subscriptionenddate::timestamp as subscr_end, - name as subscr_name, - "version#392c30e6081c24fb78ddf6d622de4f33"::integer - as subscr_version, - * - from zuora.zuora_subscription -); - -create or replace view {{env.schema}}.zuora_rate_plan as -( - select - id as rate_plan_id, - subscriptionid as subscr_id, - * - from zuora.zuora_rate_plan -); - - -create or replace view {{env.schema}}.zuora_rate_plan_charge as -( - select - rateplanid as rate_plan_id, - effectivestartdate::timestamp as rpc_start, - effectiveenddate::timestamp as rpc_end, - mrr as "@mrr", - islastsegment as rpc_last_segment, - * - from zuora.zuora_rate_plan_charge -); - - -create or replace view {{env.schema}}.zuora_amendment as -( - select - id as amend_id, - subscriptionid as subscr_id, - effectivedate::timestamp as amend_start, - * - from zuora.zuora_amendment -); - - - -create or replace view {{env.schema}}.zuora_subscriptions_w_charges_and_amendments as -( - -- get all subscriptions with possible ammendments for all accounts - with subscr_w_amendments as - ( - select - account_number, acc.account_id, sub.subscr_id, - subscr_name, subscr_status, subscr_term_type, - subscr_start, subscr_end, subscr_version, amend_id, amend_start - from {{env.schema}}.zuora_account acc - inner join {{env.schema}}.zuora_subscription sub - on acc.account_id = sub.account_id - -- add ammendments - left outer join {{env.schema}}.zuora_amendment amend - on sub.subscr_id = amend.subscr_id - ) - - select - account_number, account_id, sub.subscr_id, - subscr_name, subscr_status, subscr_term_type, - subscr_start, subscr_end, subscr_version, amend_id, amend_start, - rpc_start, rpc_end, rpc_last_segment, - min(subscr_start) over() as first_subscr, - "@mrr" as mrr - from subscr_w_amendments sub - inner join {{env.schema}}.zuora_rate_plan rp - on rp.subscr_id = sub.subscr_id - inner join {{env.schema}}.zuora_rate_plan_charge rpc - on rpc.rate_plan_id = rp.rate_plan_id -); - - - - diff --git a/model/zuora/zuora_account.sql b/model/zuora/zuora_account.sql new file mode 100644 index 0000000..233c201 --- /dev/null +++ b/model/zuora/zuora_account.sql @@ -0,0 +1,5 @@ +select + id as account_id, + accountnumber as account_number, + * +from zuora.zuora_account diff --git a/model/zuora/zuora_amendment.sql b/model/zuora/zuora_amendment.sql new file mode 100644 index 0000000..567b2e9 --- /dev/null +++ b/model/zuora/zuora_amendment.sql @@ -0,0 +1,6 @@ +select + id as amend_id, + subscriptionid as subscr_id, + effectivedate::timestamp as amend_start, + * +from zuora.zuora_amendment diff --git a/model/zuora/zuora_rate_plan.sql b/model/zuora/zuora_rate_plan.sql new file mode 100644 index 0000000..e68af44 --- /dev/null +++ b/model/zuora/zuora_rate_plan.sql @@ -0,0 +1,5 @@ +select + id as rate_plan_id, + subscriptionid as subscr_id, + * +from zuora.zuora_rate_plan diff --git a/model/zuora/zuora_rate_plan_charge.sql b/model/zuora/zuora_rate_plan_charge.sql new file mode 100644 index 0000000..eecf00d --- /dev/null +++ b/model/zuora/zuora_rate_plan_charge.sql @@ -0,0 +1,8 @@ +select + rateplanid as rate_plan_id, + effectivestartdate::timestamp as rpc_start, + effectiveenddate::timestamp as rpc_end, + mrr as "@mrr", + islastsegment as rpc_last_segment, + * +from zuora.zuora_rate_plan_charge diff --git a/model/zuora/zuora_subscription.sql b/model/zuora/zuora_subscription.sql new file mode 100644 index 0000000..ecae2de --- /dev/null +++ b/model/zuora/zuora_subscription.sql @@ -0,0 +1,12 @@ +select + id as subscr_id, + status as subscr_status, + termtype as subscr_term_type, + accountid as account_id, + contracteffectivedate::timestamp as subscr_start, + subscriptionenddate::timestamp as subscr_end, + name as subscr_name, + "version#392c30e6081c24fb78ddf6d622de4f33"::integer + as subscr_version, + * +from zuora.zuora_subscription diff --git a/model/zuora/zuora_subscriptions_w_charges_and_amendments.sql b/model/zuora/zuora_subscriptions_w_charges_and_amendments.sql new file mode 100644 index 0000000..4da0efa --- /dev/null +++ b/model/zuora/zuora_subscriptions_w_charges_and_amendments.sql @@ -0,0 +1,27 @@ +-- get all subscriptions with possible ammendments for all accounts +with subscr_w_amendments as +( + select + account_number, acc.account_id, sub.subscr_id, + subscr_name, subscr_status, subscr_term_type, + subscr_start, subscr_end, subscr_version, amend_id, amend_start + from {{env.schema}}.zuora_account acc + inner join {{env.schema}}.zuora_subscription sub + on acc.account_id = sub.account_id + -- add ammendments + left outer join {{env.schema}}.zuora_amendment amend + on sub.subscr_id = amend.subscr_id +) + +select + account_number, account_id, sub.subscr_id, + subscr_name, subscr_status, subscr_term_type, + subscr_start, subscr_end, subscr_version, amend_id, amend_start, + rpc_start, rpc_end, rpc_last_segment, + min(subscr_start) over() as first_subscr, + "@mrr" as mrr +from subscr_w_amendments sub +inner join {{env.schema}}.zuora_rate_plan rp + on rp.subscr_id = sub.subscr_id +inner join {{env.schema}}.zuora_rate_plan_charge rpc + on rpc.rate_plan_id = rp.rate_plan_id From 7c799d8650834cc5a94419bd259fb9a5eec3f217 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Fri, 25 Mar 2016 16:52:01 -0400 Subject: [PATCH 49/88] update dbt project file --- dbt_project.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 78eefc0..381e2ca 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -6,8 +6,8 @@ clean-targets: ["target"] # directories removed by the clean task # Run configuration # output environments -models: ['pardot'] -table_or_view: view +#models: ['*'] +#table_or_view: table run-target: my_redshift From e6dff227f85943b66874ee51031d966b846910a0 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Fri, 25 Mar 2016 16:53:47 -0400 Subject: [PATCH 50/88] updates based on code review --- model/stripe/model.sql | 67 ++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 41 deletions(-) diff --git a/model/stripe/model.sql b/model/stripe/model.sql index ce2a667..e6d70e0 100644 --- a/model/stripe/model.sql +++ b/model/stripe/model.sql @@ -62,15 +62,15 @@ create or replace view {{env.schema}}.stripe_invoices_transformed as ( where paid is true and forgiven is false - ), d1 as ( + ), days as ( select (min(period_start) over () + row_number() over ())::date as date_day from invoices - ), dates as ( + ), months as ( select distinct date_trunc('month', date_day)::date as date_month - from d1 + from days where date_day <= current_date ), customers as ( @@ -78,55 +78,40 @@ create or replace view {{env.schema}}.stripe_invoices_transformed as ( select customer, min(period_start) as active_from, max(period_end) as active_to from invoices where period_start <= current_date - group by 1 + group by customer ), customer_dates as ( - select date_month, customer - from dates d + select m.date_month, c.customer + from months m inner join customers c - on d.date_month >= date_trunc('month', c.active_from) - and d.date_month < date_trunc('month', c.active_to) - - ), data as ( - - select date_month, d.customer, period_start, period_end, - "interval" as period, - case "interval" - when 'yearly' - then coalesce(i.total, 0)::float / 12 / 100 - else - coalesce(i.total, 0)::float / 100 - end as total, - first_value(date_month) - over (partition by d.customer - order by date_month - rows between unbounded preceding and unbounded following - ) as first_purchase_month, - last_value(date_month) - over (partition by d.customer - order by date_month - rows between unbounded preceding and unbounded following - ) as last_purchase_month - from customer_dates d - left outer join invoices i - on d.date_month >= date_trunc('month', i.period_start) - and d.date_month < date_trunc('month', i.period_end) - and d.customer = i.customer - left outer join {{env.schema}}.stripe_subscriptions s on i.subscription_id = s.id - left outer join {{env.schema}}.stripe_plans p on s.plan_id = p.id + on m.date_month >= date_trunc('month', c.active_from) + and m.date_month < date_trunc('month', c.active_to) ) - select customer, date_month, total, period_end, period, - case first_purchase_month + select date_month, d.customer, period_start, period_end, + "interval" as period, + case "interval" + when 'yearly' + then coalesce(i.total, 0)::float / 12 / 100 + else + coalesce(i.total, 0)::float / 100 + end as total, + case min(date_month) over(partition by d.customer) when date_month then 1 else 0 - end as first_payment, - case last_purchase_month + end as first_payment, + case max(date_month) over(partition by d.customer) when date_month then 1 else 0 end as last_payment - from data + from customer_dates d + left outer join invoices i + on d.date_month >= date_trunc('month', i.period_start) + and d.date_month < date_trunc('month', i.period_end) + and d.customer = i.customer + left outer join {{env.schema}}.stripe_subscriptions s on i.subscription_id = s.id + left outer join {{env.schema}}.stripe_plans p on s.plan_id = p.id ); From 677e5864a24cb7db80bd76576e4149fe87df2fc3 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Fri, 25 Mar 2016 16:58:47 -0400 Subject: [PATCH 51/88] final CR comments addressed --- analysis/stripe/analysis.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/analysis/stripe/analysis.sql b/analysis/stripe/analysis.sql index 5763c4c..a6630e3 100644 --- a/analysis/stripe/analysis.sql +++ b/analysis/stripe/analysis.sql @@ -10,11 +10,11 @@ with invoices as ( ), plan_changes as ( select - i.*, + *, lag(total) over (partition by customer order by date_month) as prior_month_total, - i.total - coalesce(lag(total) over (partition by customer order by date_month), 0) as change, + total - coalesce(lag(total) over (partition by customer order by date_month), 0) as change, lag(period_end) over (partition by customer order by date_month) as prior_month_period_end - from invoices i + from invoices ), data as ( From 2b3ca791139d7330784cb5e9371e5ee5e5f1f8d2 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Mon, 28 Mar 2016 10:35:31 -0400 Subject: [PATCH 52/88] update configs to render to table --- dbt_project.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 381e2ca..a72be0e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -6,8 +6,8 @@ clean-targets: ["target"] # directories removed by the clean task # Run configuration # output environments -#models: ['*'] -#table_or_view: table +models: ['*'] +table_or_view: table run-target: my_redshift From 7e52da55240b1ae041e3f953143df68bdb9861b5 Mon Sep 17 00:00:00 2001 From: jthandy Date: Mon, 28 Mar 2016 21:22:38 -0400 Subject: [PATCH 53/88] Create License.md --- License.md | 201 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 License.md diff --git a/License.md b/License.md new file mode 100644 index 0000000..8dada3e --- /dev/null +++ b/License.md @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From b81947a3372ec51617b2b79db843c2e0eff733d0 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 29 Mar 2016 14:35:30 -0400 Subject: [PATCH 54/88] change emails to pardot_emails (for now) --- model/email/emails_denormalized.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/email/emails_denormalized.sql b/model/email/emails_denormalized.sql index 7acc8f1..a483de7 100644 --- a/model/email/emails_denormalized.sql +++ b/model/email/emails_denormalized.sql @@ -1,6 +1,6 @@ with events as ( - select * from {{env.schema}}.emails + select * from {{env.schema}}.pardot_emails ), From fa314ee32bc863983151d797581770077a895712 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Tue, 29 Mar 2016 15:02:56 -0400 Subject: [PATCH 55/88] move stripe models folder --- {model => models}/stripe/model.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {model => models}/stripe/model.sql (100%) diff --git a/model/stripe/model.sql b/models/stripe/model.sql similarity index 100% rename from model/stripe/model.sql rename to models/stripe/model.sql From f3fef6aeda1a215941f6393052d74c631348990e Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 29 Mar 2016 15:45:07 -0400 Subject: [PATCH 56/88] fix master merge (model --> models) --- models/email/emails_denormalized.sql | 36 ++++++ models/email/model.sql | 40 ------ models/magento/model.sql | 16 --- models/magento/order_items.sql | 3 + models/magento/products.sql | 4 + models/pardot/model.sql | 119 ------------------ models/pardot/pardot_emails.sql | 9 ++ models/pardot/pardot_tests.sql | 20 +++ models/pardot/pardot_visitoractivity.sql | 15 +++ .../pardot_visitoractivity_events_meta.sql | 35 ++++++ .../pardot_visitoractivity_types_meta.sql | 37 ++++++ models/pardot/tests.sql | 26 ---- models/segment/model.sql | 12 -- models/segment/segment_track.sql | 8 ++ models/snowplow/model.sql | 11 -- models/snowplow/snowplow_events.sql | 7 ++ models/stripe/stripe_invoices.sql | 14 +++ models/stripe/stripe_invoices_cleaned.sql | 14 +++ ...el.sql => stripe_invoices_transformed.sql} | 58 +-------- models/stripe/stripe_plans.sql | 8 ++ models/stripe/stripe_subscriptions.sql | 8 ++ models/trello/model.sql | 20 --- models/trello/test.sql | 25 ---- models/trello/trello_card_location.sql | 18 +++ models/trello/trello_model_tests.sql | 21 ++++ models/zuora/model.sql | 95 -------------- models/zuora/zuora_account.sql | 5 + models/zuora/zuora_amendment.sql | 6 + models/zuora/zuora_rate_plan.sql | 5 + models/zuora/zuora_rate_plan_charge.sql | 8 ++ models/zuora/zuora_subscription.sql | 12 ++ ...subscriptions_w_charges_and_amendments.sql | 27 ++++ 32 files changed, 321 insertions(+), 421 deletions(-) create mode 100644 models/email/emails_denormalized.sql delete mode 100644 models/email/model.sql delete mode 100644 models/magento/model.sql create mode 100644 models/magento/order_items.sql create mode 100644 models/magento/products.sql delete mode 100644 models/pardot/model.sql create mode 100644 models/pardot/pardot_emails.sql create mode 100644 models/pardot/pardot_tests.sql create mode 100644 models/pardot/pardot_visitoractivity.sql create mode 100644 models/pardot/pardot_visitoractivity_events_meta.sql create mode 100644 models/pardot/pardot_visitoractivity_types_meta.sql delete mode 100644 models/pardot/tests.sql delete mode 100644 models/segment/model.sql create mode 100644 models/segment/segment_track.sql delete mode 100644 models/snowplow/model.sql create mode 100644 models/snowplow/snowplow_events.sql create mode 100644 models/stripe/stripe_invoices.sql create mode 100644 models/stripe/stripe_invoices_cleaned.sql rename models/stripe/{model.sql => stripe_invoices_transformed.sql} (62%) create mode 100644 models/stripe/stripe_plans.sql create mode 100644 models/stripe/stripe_subscriptions.sql delete mode 100644 models/trello/model.sql delete mode 100644 models/trello/test.sql create mode 100644 models/trello/trello_card_location.sql create mode 100644 models/trello/trello_model_tests.sql delete mode 100644 models/zuora/model.sql create mode 100644 models/zuora/zuora_account.sql create mode 100644 models/zuora/zuora_amendment.sql create mode 100644 models/zuora/zuora_rate_plan.sql create mode 100644 models/zuora/zuora_rate_plan_charge.sql create mode 100644 models/zuora/zuora_subscription.sql create mode 100644 models/zuora/zuora_subscriptions_w_charges_and_amendments.sql diff --git a/models/email/emails_denormalized.sql b/models/email/emails_denormalized.sql new file mode 100644 index 0000000..a483de7 --- /dev/null +++ b/models/email/emails_denormalized.sql @@ -0,0 +1,36 @@ +with events as ( + + select * from {{env.schema}}.pardot_emails + +), + +sends as ( + + select "@user_id", "@timestamp", "@email_id" + from events + where "@event" = 'email sent' + +), + +opens as ( + + select "@user_id", "@timestamp", "@email_id" + from events + where "@event" = 'email opened' + +), clicks as ( + + select "@user_id", "@timestamp", "@email_id" + from events + where "@event" = 'email click' + +) + +select s."@user_id", s."@timestamp" as sent_timestamp, o."@timestamp" as opened_timestamp, c."@timestamp" as clicked_timestamp, + decode(o."@timestamp", null, 0, 1) as "opened?", + decode(c."@timestamp", null, 0, 1) as "clicked?", + row_number() over (partition by s."@user_id" order by s."@timestamp") as email_number +from sends s + left outer join opens o on s."@email_id" = o."@email_id" + left outer join clicks c on s."@email_id" = c."@email_id" +order by 1, 2 diff --git a/models/email/model.sql b/models/email/model.sql deleted file mode 100644 index 0365b95..0000000 --- a/models/email/model.sql +++ /dev/null @@ -1,40 +0,0 @@ -create or replace view {{env.schema}}.emails_denormalized as ( - - with events as ( - - select * from {{env.schema}}.emails - - ), - - sends as ( - - select "@user_id", "@timestamp", "@email_id" - from events - where "@event" = 'email sent' - - ), - - opens as ( - - select "@user_id", "@timestamp", "@email_id" - from events - where "@event" = 'email opened' - - ), clicks as ( - - select "@user_id", "@timestamp", "@email_id" - from events - where "@event" = 'email click' - - ) - - select s."@user_id", s."@timestamp" as sent_timestamp, o."@timestamp" as opened_timestamp, c."@timestamp" as clicked_timestamp, - decode(o."@timestamp", null, 0, 1) as "opened?", - decode(c."@timestamp", null, 0, 1) as "clicked?", - row_number() over (partition by s."@user_id" order by s."@timestamp") as email_number - from sends s - left outer join opens o on s."@email_id" = o."@email_id" - left outer join clicks c on s."@email_id" = c."@email_id" - order by 1, 2 - -); diff --git a/models/magento/model.sql b/models/magento/model.sql deleted file mode 100644 index 227e385..0000000 --- a/models/magento/model.sql +++ /dev/null @@ -1,16 +0,0 @@ -create or replace view {{env.schema}}.magento_order_items as ( - - select * - FROM - sample_magento_database.sales_flat_order_item - -); - - -create or replace view {{env.schema}}.magento_products as ( - - select * - FROM - sample_magento_database.catalog_product_flat_1 - -); diff --git a/models/magento/order_items.sql b/models/magento/order_items.sql new file mode 100644 index 0000000..07d33d3 --- /dev/null +++ b/models/magento/order_items.sql @@ -0,0 +1,3 @@ +select * +FROM + sample_magento_database.sales_flat_order_item diff --git a/models/magento/products.sql b/models/magento/products.sql new file mode 100644 index 0000000..87468fb --- /dev/null +++ b/models/magento/products.sql @@ -0,0 +1,4 @@ +select * +FROM + sample_magento_database.catalog_product_flat_1 + diff --git a/models/pardot/model.sql b/models/pardot/model.sql deleted file mode 100644 index 2f78768..0000000 --- a/models/pardot/model.sql +++ /dev/null @@ -1,119 +0,0 @@ -create or replace view {{env.schema}}.pardot_visitoractivity_types_meta as ( - - --these literal values are pulled from pardot's api docs here: - --http://developer.pardot.com/kb/object-field-references/#visitor-activity - --they change periodically over time and this query will need to be correspondingly modified. - select 1 as type, 'Click' as type_decoded union all - select 2, 'View' union all - select 3, 'Error' union all - select 4, 'Success' union all - select 5, 'Session' union all - select 6, 'Sent' union all - select 7, 'Search' union all - select 8, 'New Opportunity' union all - select 9, 'Opportunity Won' union all - select 10, 'Opportunity Lost' union all - select 11, 'Open' union all - select 12, 'Unsubscribe Page' union all - select 13, 'Bounced' union all - select 14, 'Spam Complaint' union all - select 15, 'Email Preference Page' union all - select 16, 'Resubscribed' union all - select 17, 'Click (Third Party)' union all - select 18, 'Opportunity Reopened' union all - select 19, 'Opportunity Linked' union all - select 20, 'Visit' union all - select 21, 'Custom URL click' union all - select 22, 'Olark Chat' union all - select 23, 'Invited to Webinar' union all - select 24, 'Attended Webinar' union all - select 25, 'Registered for Webinar' union all - select 26, 'Social Post Click' union all - select 27, 'Video View' union all - select 28, 'Event Registered' union all - select 29, 'Event Checked In' union all - select 30, 'Video Conversion' union all - select 31, 'UserVoice Suggestion' union all - select 32, 'UserVoice Comment' union all - select 33, 'UserVoice Ticket' union all - select 34, 'Video Watched (>= 75% watched)' - -); - - - -create or replace view {{env.schema}}.pardot_visitoractivity_events_meta as ( - - --even with the type decoding that Pardot specifically provides, actually what is going on in a given event - --is somewhat ambiguous. this is an attempt to map type and type_name to a more event-based "event action" field - --which is always written in more standard action-oriented terms. - select 22 as "type", 'Chat Transcript' as type_name, 'chatted via olark' as event_name union all - select 21, 'Custom Redirect', 'clicked a custom redirect' union all - select 6, 'Email', 'email sent' union all - select 11, 'Email', 'email opened' union all - select 13, 'Email', 'email bounced' union all - select 14, 'Email', 'email reported spam' union all - select 1, 'Email Tracker', 'email click' union all - select 28, 'Event', 'registered for event' union all - select 29, 'Event', 'checked in at event' union all - select 2, 'File', 'viewed a file' union all - select 3, 'Form', 'submitted a form with an error' union all - select 2, 'Form', 'viewed a form' union all - select 4, 'Form', 'successfully submitted a form' union all - select 4, 'Form Handler', 'successfully submitted a form handler' union all - select 2, 'Landing Page', 'viewed a landing page' union all - select 4, 'Landing Page', 'successfully submitted the form on a landing page' union all - select 3, 'Landing Page', 'submitted the form on a landing page with an error' union all - select 2, 'Multivariate Landing Page', 'viewed multivariate landing page' union all - select 4, 'Multivariate Landing Page', 'successfully submitted multivariate landing page' union all - select 3, 'Multivariate Landing Page', 'submitted multivariate landing page with an error' union all - select 8, 'New Opportunity', 'opened opportunity' union all - select 19, 'Opportunity Associated', 'linked existing opportunity' union all - select 10, 'Opportunity Lost', 'lost opportunity' union all - select 9, 'Opportunity Won', 'won opportunity' union all - select 2, 'Page View', 'viewed highlighted page' union all - select 34, 'Video', 'watched 75% or more of video' union all - select 27, 'Video', 'watched video' union all - select 30, 'Video', 'converted from video call to action' union all - select 20, 'Visit', 'visited website' union all - select 25, 'Webinar', 'registered for webinar' union all - select 24, 'Webinar', 'attended webinar' union all - select 18, '', 'reopened opportunity' - -); - - -create or replace view {{env.schema}}.pardot_visitoractivity as ( - --this table has a bunch of types that really should be event actions but are very poorly formulated. - --the custom logic in this view is an attempt to fix that. - --not all of the various type / type_name combinations have been accounted for yet; I still need to determine exactly what some of them mean. - select - -- event_stream interface - va.created_at as "@timestamp", - e.event_name as "@event", - va.prospect_id as "@user_id", - va.* - from - olga_pardot.visitoractivity va - inner join {{env.schema}}.pardot_visitoractivity_events_meta e - on va."type" = e."type" and va.type_name = e.type_name - inner join {{env.schema}}.pardot_visitoractivity_types_meta t - on va."type" = t."type" -); - -COMMENT ON VIEW {{env.schema}}.pardot_visitoractivity IS 'timeseries,funnel,cohort'; - - -/* -This model maps pardot data from the visitoractivity table to the email analysis interface. -It conforms to the basic email interface, not the extended email interface, because Pardot does not supply -data necessary to conform to the extended interface. -*/ - -create or replace view {{env.schema}}.emails as ( - - select "@timestamp", "@event", "@user_id", email_id as "@email_id", details as "@subject" - from {{env.schema}}.pardot_visitoractivity - where "@event" in ('email sent', 'email opened', 'email click') - -); diff --git a/models/pardot/pardot_emails.sql b/models/pardot/pardot_emails.sql new file mode 100644 index 0000000..b8308f1 --- /dev/null +++ b/models/pardot/pardot_emails.sql @@ -0,0 +1,9 @@ +/* +This model maps pardot data from the visitoractivity table to the email analysis interface. +It conforms to the basic email interface, not the extended email interface, because Pardot does not supply +data necessary to conform to the extended interface. +*/ + +select "@timestamp", "@event", "@user_id", email_id as "@email_id", details as "@subject" + from {{env.schema}}.pardot_visitoractivity +where "@event" in ('email sent', 'email opened', 'email click') diff --git a/models/pardot/pardot_tests.sql b/models/pardot/pardot_tests.sql new file mode 100644 index 0000000..20155df --- /dev/null +++ b/models/pardot/pardot_tests.sql @@ -0,0 +1,20 @@ +select + 'visitoractivity_fresher_than_one_day' as name, + 'Most recent visitoractivity entry is no more than one day old' as description, + max("@timestamp"::timestamp) > current_date - '1 day'::interval as result +from {{env.schema}}.pardot_visitoractivity + + + + + + + +/* + +Other tests I want to do: +- make sure there are records from every day since the first day we see any records +- make sure all prospect ids from visitoractivity show up in prospects +- make sure there are no unmapped types + +*/ diff --git a/models/pardot/pardot_visitoractivity.sql b/models/pardot/pardot_visitoractivity.sql new file mode 100644 index 0000000..f56509a --- /dev/null +++ b/models/pardot/pardot_visitoractivity.sql @@ -0,0 +1,15 @@ +--this table has a bunch of types that really should be event actions but are very poorly formulated. +--the custom logic in this view is an attempt to fix that. +--not all of the various type / type_name combinations have been accounted for yet; I still need to determine exactly what some of them mean. +select + -- event_stream interface + va.created_at as "@timestamp", + e.event_name as "@event", + va.prospect_id as "@user_id", + va.* +from + olga_pardot.visitoractivity va + inner join {{env.schema}}.pardot_visitoractivity_events_meta e + on va."type" = e."type" and va.type_name = e.type_name + inner join {{env.schema}}.pardot_visitoractivity_types_meta t + on va."type" = t."type" diff --git a/models/pardot/pardot_visitoractivity_events_meta.sql b/models/pardot/pardot_visitoractivity_events_meta.sql new file mode 100644 index 0000000..2339b65 --- /dev/null +++ b/models/pardot/pardot_visitoractivity_events_meta.sql @@ -0,0 +1,35 @@ +--even with the type decoding that Pardot specifically provides, actually what is going on in a given event +--is somewhat ambiguous. this is an attempt to map type and type_name to a more event-based "event action" field +--which is always written in more standard action-oriented terms. +select 22 as "type", 'Chat Transcript' as type_name, 'chatted via olark' as event_name union all +select 21, 'Custom Redirect', 'clicked a custom redirect' union all +select 6, 'Email', 'email sent' union all +select 11, 'Email', 'email opened' union all +select 13, 'Email', 'email bounced' union all +select 14, 'Email', 'email reported spam' union all +select 1, 'Email Tracker', 'email click' union all +select 28, 'Event', 'registered for event' union all +select 29, 'Event', 'checked in at event' union all +select 2, 'File', 'viewed a file' union all +select 3, 'Form', 'submitted a form with an error' union all +select 2, 'Form', 'viewed a form' union all +select 4, 'Form', 'successfully submitted a form' union all +select 4, 'Form Handler', 'successfully submitted a form handler' union all +select 2, 'Landing Page', 'viewed a landing page' union all +select 4, 'Landing Page', 'successfully submitted the form on a landing page' union all +select 3, 'Landing Page', 'submitted the form on a landing page with an error' union all +select 2, 'Multivariate Landing Page', 'viewed multivariate landing page' union all +select 4, 'Multivariate Landing Page', 'successfully submitted multivariate landing page' union all +select 3, 'Multivariate Landing Page', 'submitted multivariate landing page with an error' union all +select 8, 'New Opportunity', 'opened opportunity' union all +select 19, 'Opportunity Associated', 'linked existing opportunity' union all +select 10, 'Opportunity Lost', 'lost opportunity' union all +select 9, 'Opportunity Won', 'won opportunity' union all +select 2, 'Page View', 'viewed highlighted page' union all +select 34, 'Video', 'watched 75% or more of video' union all +select 27, 'Video', 'watched video' union all +select 30, 'Video', 'converted from video call to action' union all +select 20, 'Visit', 'visited website' union all +select 25, 'Webinar', 'registered for webinar' union all +select 24, 'Webinar', 'attended webinar' union all +select 18, '', 'reopened opportunity' diff --git a/models/pardot/pardot_visitoractivity_types_meta.sql b/models/pardot/pardot_visitoractivity_types_meta.sql new file mode 100644 index 0000000..0e664bf --- /dev/null +++ b/models/pardot/pardot_visitoractivity_types_meta.sql @@ -0,0 +1,37 @@ +--these literal values are pulled from pardot's api docs here: +--http://developer.pardot.com/kb/object-field-references/#visitor-activity +--they change periodically over time and this query will need to be correspondingly modified. +select 1 as type, 'Click' as type_decoded union all +select 2, 'View' union all +select 3, 'Error' union all +select 4, 'Success' union all +select 5, 'Session' union all +select 6, 'Sent' union all +select 7, 'Search' union all +select 8, 'New Opportunity' union all +select 9, 'Opportunity Won' union all +select 10, 'Opportunity Lost' union all +select 11, 'Open' union all +select 12, 'Unsubscribe Page' union all +select 13, 'Bounced' union all +select 14, 'Spam Complaint' union all +select 15, 'Email Preference Page' union all +select 16, 'Resubscribed' union all +select 17, 'Click (Third Party)' union all +select 18, 'Opportunity Reopened' union all +select 19, 'Opportunity Linked' union all +select 20, 'Visit' union all +select 21, 'Custom URL click' union all +select 22, 'Olark Chat' union all +select 23, 'Invited to Webinar' union all +select 24, 'Attended Webinar' union all +select 25, 'Registered for Webinar' union all +select 26, 'Social Post Click' union all +select 27, 'Video View' union all +select 28, 'Event Registered' union all +select 29, 'Event Checked In' union all +select 30, 'Video Conversion' union all +select 31, 'UserVoice Suggestion' union all +select 32, 'UserVoice Comment' union all +select 33, 'UserVoice Ticket' union all +select 34, 'Video Watched (>= 75% watched)' diff --git a/models/pardot/tests.sql b/models/pardot/tests.sql deleted file mode 100644 index 092c605..0000000 --- a/models/pardot/tests.sql +++ /dev/null @@ -1,26 +0,0 @@ -create or replace view {{env.schema}}.pardot_model_tests - (name, description, result) - as ( - - select - 'visitoractivity_fresher_than_one_day', - 'Most recent visitoractivity entry is no more than one day old', - max("@timestamp"::timestamp) > current_date - '1 day'::interval - from {{env.schema}}.pardot_visitoractivity - - ); - - - - - - - - /* - -Other tests I want to do: - - make sure there are records from every day since the first day we see any records - - make sure all prospect ids from visitoractivity show up in prospects - - make sure there are no unmapped types - - */ diff --git a/models/segment/model.sql b/models/segment/model.sql deleted file mode 100644 index 7a10851..0000000 --- a/models/segment/model.sql +++ /dev/null @@ -1,12 +0,0 @@ -create or replace view {{env.schema}}.segment_track as ( - select - "timestamp"::timestamp as "@timestamp", - "event" as "@event", - "userid" as "@user_id", - * - - from - segment.track -); - -comment on view {{env.schema}}.segment_track is 'timeseries,funnel,cohort'; diff --git a/models/segment/segment_track.sql b/models/segment/segment_track.sql new file mode 100644 index 0000000..b393fea --- /dev/null +++ b/models/segment/segment_track.sql @@ -0,0 +1,8 @@ +select + "timestamp"::timestamp as "@timestamp", + "event" as "@event", + "userid" as "@user_id", + * + +from + segment.track diff --git a/models/snowplow/model.sql b/models/snowplow/model.sql deleted file mode 100644 index e81a581..0000000 --- a/models/snowplow/model.sql +++ /dev/null @@ -1,11 +0,0 @@ -create or replace view {{env.schema}}.snowplow_events as ( - select - "collector_tstamp" as "@timestamp", - "event_name" as "@event", - "domain_userid" as "@user_id", - * - from - atomic.events -); - -comment on view {{env.schema}}.snowplow_events is 'timeseries,funnel,cohort'; diff --git a/models/snowplow/snowplow_events.sql b/models/snowplow/snowplow_events.sql new file mode 100644 index 0000000..f438b35 --- /dev/null +++ b/models/snowplow/snowplow_events.sql @@ -0,0 +1,7 @@ +select + "collector_tstamp" as "@timestamp", + "event_name" as "@event", + "domain_userid" as "@user_id", + * +from + atomic.events diff --git a/models/stripe/stripe_invoices.sql b/models/stripe/stripe_invoices.sql new file mode 100644 index 0000000..14a5f74 --- /dev/null +++ b/models/stripe/stripe_invoices.sql @@ -0,0 +1,14 @@ + + select + customer, + date, + forgiven, + subscription as subscription_id, + paid, + total, + period_start, + period_end + from + demo_data.stripe_invoices + + diff --git a/models/stripe/stripe_invoices_cleaned.sql b/models/stripe/stripe_invoices_cleaned.sql new file mode 100644 index 0000000..74dfbdc --- /dev/null +++ b/models/stripe/stripe_invoices_cleaned.sql @@ -0,0 +1,14 @@ + + select + customer, + timestamp 'epoch' + date * interval '1 Second' as date, + forgiven, + subscription_id, + paid, + total, + timestamp 'epoch' + period_start * interval '1 Second' as period_start, + timestamp 'epoch' + period_end * interval '1 Second' as period_end + from + {{env.schema}}.stripe_invoices + + diff --git a/models/stripe/model.sql b/models/stripe/stripe_invoices_transformed.sql similarity index 62% rename from models/stripe/model.sql rename to models/stripe/stripe_invoices_transformed.sql index e6d70e0..fd6b48f 100644 --- a/models/stripe/model.sql +++ b/models/stripe/stripe_invoices_transformed.sql @@ -1,59 +1,3 @@ -create or replace view {{env.schema}}.stripe_invoices as ( - - select - customer, - date, - forgiven, - subscription as subscription_id, - paid, - total, - period_start, - period_end - from - demo_data.stripe_invoices - -); - -create or replace view {{env.schema}}.stripe_invoices_cleaned as ( - - select - customer, - timestamp 'epoch' + date * interval '1 Second' as date, - forgiven, - subscription_id, - paid, - total, - timestamp 'epoch' + period_start * interval '1 Second' as period_start, - timestamp 'epoch' + period_end * interval '1 Second' as period_end - from - {{env.schema}}.stripe_invoices - -); - -create or replace view {{env.schema}}.stripe_subscriptions as ( - - select - id, - plan__id as plan_id - from - demo_data.stripe_subscriptions - -); - -create or replace view {{env.schema}}.stripe_plans as ( - - select - id, - interval - from - demo_data.stripe_plans - -); - - - - -create or replace view {{env.schema}}.stripe_invoices_transformed as ( with invoices as ( @@ -114,4 +58,4 @@ create or replace view {{env.schema}}.stripe_invoices_transformed as ( left outer join {{env.schema}}.stripe_subscriptions s on i.subscription_id = s.id left outer join {{env.schema}}.stripe_plans p on s.plan_id = p.id -); + diff --git a/models/stripe/stripe_plans.sql b/models/stripe/stripe_plans.sql new file mode 100644 index 0000000..81a39b9 --- /dev/null +++ b/models/stripe/stripe_plans.sql @@ -0,0 +1,8 @@ + + select + id, + interval + from + demo_data.stripe_plans + + diff --git a/models/stripe/stripe_subscriptions.sql b/models/stripe/stripe_subscriptions.sql new file mode 100644 index 0000000..bb1fe44 --- /dev/null +++ b/models/stripe/stripe_subscriptions.sql @@ -0,0 +1,8 @@ + + select + id, + plan__id as plan_id + from + demo_data.stripe_subscriptions + + diff --git a/models/trello/model.sql b/models/trello/model.sql deleted file mode 100644 index 3eee29f..0000000 --- a/models/trello/model.sql +++ /dev/null @@ -1,20 +0,0 @@ -create or replace view {{env.schema}}.trello_card_location as ( - select - id, - idmembercreator, - date, - "type", - data__card__id, - data__card__name, - coalesce(data__list__id, - data__listafter__id, - lag(coalesce(data__list__id, data__listafter__id)) ignore nulls over (partition by data__card__id order by date)) as data__list__id, - coalesce(data__boardtarget__id, data__board__id) as data__board__id, - coalesce(data__card__closed, - lag(data__card__closed) ignore nulls over (partition by data__card__id order by date), - false) as data__card__closed - from trello_growth.trello_actions - where - data__card__id is not null - and "type" in ('createCard', 'updateCard', 'moveCardFromBoard', 'moveCardToBoard', 'commentCard') -); diff --git a/models/trello/test.sql b/models/trello/test.sql deleted file mode 100644 index f2d5a9b..0000000 --- a/models/trello/test.sql +++ /dev/null @@ -1,25 +0,0 @@ -create or replace view {{env.schema}}.trello_model_tests -(name, description, result) -as ( - with null_boards_or_lists as - ( - select id - from {{env.schema}}.trello_card_location - where - data__board__id is null - or data__list__id is null - ) - select - 'no_null_boards_or_lists', - 'All location entries have a non-null board and list id', - count(*) = 0 - from null_boards_or_lists - - union all - - select - 'fresher_than_one_day', - 'Most recent entry is no more than one day old', - max(date::timestamp) > current_date - '1 day'::interval - from {{env.schema}}.trello_card_location -); diff --git a/models/trello/trello_card_location.sql b/models/trello/trello_card_location.sql new file mode 100644 index 0000000..f6ebe45 --- /dev/null +++ b/models/trello/trello_card_location.sql @@ -0,0 +1,18 @@ +select + id, + idmembercreator, + date, + "type", + data__card__id, + data__card__name, + coalesce(data__list__id, + data__listafter__id, + lag(coalesce(data__list__id, data__listafter__id)) ignore nulls over (partition by data__card__id order by date)) as data__list__id, + coalesce(data__boardtarget__id, data__board__id) as data__board__id, + coalesce(data__card__closed, + lag(data__card__closed) ignore nulls over (partition by data__card__id order by date), + false) as data__card__closed +from trello_growth.trello_actions +where + data__card__id is not null + and "type" in ('createCard', 'updateCard', 'moveCardFromBoard', 'moveCardToBoard', 'commentCard') diff --git a/models/trello/trello_model_tests.sql b/models/trello/trello_model_tests.sql new file mode 100644 index 0000000..9f9840b --- /dev/null +++ b/models/trello/trello_model_tests.sql @@ -0,0 +1,21 @@ +with null_boards_or_lists as +( + select id + from {{env.schema}}.trello_card_location + where + data__board__id is null + or data__list__id is null +) +select + 'no_null_boards_or_lists' as name, + 'All location entries have a non-null board and list id' as description, +count(*) = 0 as result +from null_boards_or_lists + +union all + +select + 'fresher_than_one_day', + 'Most recent entry is no more than one day old', +max(date::timestamp) > current_date - '1 day'::interval +from {{env.schema}}.trello_card_location diff --git a/models/zuora/model.sql b/models/zuora/model.sql deleted file mode 100644 index 630e968..0000000 --- a/models/zuora/model.sql +++ /dev/null @@ -1,95 +0,0 @@ -create or replace view {{env.schema}}.zuora_account as -( - select - id as account_id, - accountnumber as account_number, - * - from zuora.zuora_account -); - - -create or replace view {{env.schema}}.zuora_subscription as -( - select - id as subscr_id, - status as subscr_status, - termtype as subscr_term_type, - accountid as account_id, - contracteffectivedate::timestamp as subscr_start, - subscriptionenddate::timestamp as subscr_end, - name as subscr_name, - "version#392c30e6081c24fb78ddf6d622de4f33"::integer - as subscr_version, - * - from zuora.zuora_subscription -); - -create or replace view {{env.schema}}.zuora_rate_plan as -( - select - id as rate_plan_id, - subscriptionid as subscr_id, - * - from zuora.zuora_rate_plan -); - - -create or replace view {{env.schema}}.zuora_rate_plan_charge as -( - select - rateplanid as rate_plan_id, - effectivestartdate::timestamp as rpc_start, - effectiveenddate::timestamp as rpc_end, - mrr as "@mrr", - islastsegment as rpc_last_segment, - * - from zuora.zuora_rate_plan_charge -); - - -create or replace view {{env.schema}}.zuora_amendment as -( - select - id as amend_id, - subscriptionid as subscr_id, - effectivedate::timestamp as amend_start, - * - from zuora.zuora_amendment -); - - - -create or replace view {{env.schema}}.zuora_subscriptions_w_charges_and_amendments as -( - -- get all subscriptions with possible ammendments for all accounts - with subscr_w_amendments as - ( - select - account_number, acc.account_id, sub.subscr_id, - subscr_name, subscr_status, subscr_term_type, - subscr_start, subscr_end, subscr_version, amend_id, amend_start - from {{env.schema}}.zuora_account acc - inner join {{env.schema}}.zuora_subscription sub - on acc.account_id = sub.account_id - -- add ammendments - left outer join {{env.schema}}.zuora_amendment amend - on sub.subscr_id = amend.subscr_id - ) - - select - account_number, account_id, sub.subscr_id, - subscr_name, subscr_status, subscr_term_type, - subscr_start, subscr_end, subscr_version, amend_id, amend_start, - rpc_start, rpc_end, rpc_last_segment, - min(subscr_start) over() as first_subscr, - "@mrr" as mrr - from subscr_w_amendments sub - inner join {{env.schema}}.zuora_rate_plan rp - on rp.subscr_id = sub.subscr_id - inner join {{env.schema}}.zuora_rate_plan_charge rpc - on rpc.rate_plan_id = rp.rate_plan_id -); - - - - diff --git a/models/zuora/zuora_account.sql b/models/zuora/zuora_account.sql new file mode 100644 index 0000000..233c201 --- /dev/null +++ b/models/zuora/zuora_account.sql @@ -0,0 +1,5 @@ +select + id as account_id, + accountnumber as account_number, + * +from zuora.zuora_account diff --git a/models/zuora/zuora_amendment.sql b/models/zuora/zuora_amendment.sql new file mode 100644 index 0000000..567b2e9 --- /dev/null +++ b/models/zuora/zuora_amendment.sql @@ -0,0 +1,6 @@ +select + id as amend_id, + subscriptionid as subscr_id, + effectivedate::timestamp as amend_start, + * +from zuora.zuora_amendment diff --git a/models/zuora/zuora_rate_plan.sql b/models/zuora/zuora_rate_plan.sql new file mode 100644 index 0000000..e68af44 --- /dev/null +++ b/models/zuora/zuora_rate_plan.sql @@ -0,0 +1,5 @@ +select + id as rate_plan_id, + subscriptionid as subscr_id, + * +from zuora.zuora_rate_plan diff --git a/models/zuora/zuora_rate_plan_charge.sql b/models/zuora/zuora_rate_plan_charge.sql new file mode 100644 index 0000000..eecf00d --- /dev/null +++ b/models/zuora/zuora_rate_plan_charge.sql @@ -0,0 +1,8 @@ +select + rateplanid as rate_plan_id, + effectivestartdate::timestamp as rpc_start, + effectiveenddate::timestamp as rpc_end, + mrr as "@mrr", + islastsegment as rpc_last_segment, + * +from zuora.zuora_rate_plan_charge diff --git a/models/zuora/zuora_subscription.sql b/models/zuora/zuora_subscription.sql new file mode 100644 index 0000000..ecae2de --- /dev/null +++ b/models/zuora/zuora_subscription.sql @@ -0,0 +1,12 @@ +select + id as subscr_id, + status as subscr_status, + termtype as subscr_term_type, + accountid as account_id, + contracteffectivedate::timestamp as subscr_start, + subscriptionenddate::timestamp as subscr_end, + name as subscr_name, + "version#392c30e6081c24fb78ddf6d622de4f33"::integer + as subscr_version, + * +from zuora.zuora_subscription diff --git a/models/zuora/zuora_subscriptions_w_charges_and_amendments.sql b/models/zuora/zuora_subscriptions_w_charges_and_amendments.sql new file mode 100644 index 0000000..4da0efa --- /dev/null +++ b/models/zuora/zuora_subscriptions_w_charges_and_amendments.sql @@ -0,0 +1,27 @@ +-- get all subscriptions with possible ammendments for all accounts +with subscr_w_amendments as +( + select + account_number, acc.account_id, sub.subscr_id, + subscr_name, subscr_status, subscr_term_type, + subscr_start, subscr_end, subscr_version, amend_id, amend_start + from {{env.schema}}.zuora_account acc + inner join {{env.schema}}.zuora_subscription sub + on acc.account_id = sub.account_id + -- add ammendments + left outer join {{env.schema}}.zuora_amendment amend + on sub.subscr_id = amend.subscr_id +) + +select + account_number, account_id, sub.subscr_id, + subscr_name, subscr_status, subscr_term_type, + subscr_start, subscr_end, subscr_version, amend_id, amend_start, + rpc_start, rpc_end, rpc_last_segment, + min(subscr_start) over() as first_subscr, + "@mrr" as mrr +from subscr_w_amendments sub +inner join {{env.schema}}.zuora_rate_plan rp + on rp.subscr_id = sub.subscr_id +inner join {{env.schema}}.zuora_rate_plan_charge rpc + on rpc.rate_plan_id = rp.rate_plan_id From 934c6b20d729e6af7e14c2177f6022d7a2ca2c15 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 29 Mar 2016 15:45:31 -0400 Subject: [PATCH 57/88] delete model directory --- model/email/emails_denormalized.sql | 36 ------------------ model/magento/order_items.sql | 3 -- model/magento/products.sql | 4 -- model/pardot/pardot_emails.sql | 9 ----- model/pardot/pardot_tests.sql | 20 ---------- model/pardot/pardot_visitoractivity.sql | 15 -------- .../pardot_visitoractivity_events_meta.sql | 35 ------------------ .../pardot_visitoractivity_types_meta.sql | 37 ------------------- model/segment/segment_track.sql | 8 ---- model/snowplow/snowplow_events.sql | 7 ---- model/trello/trello_card_location.sql | 18 --------- model/trello/trello_model_tests.sql | 21 ----------- model/zuora/zuora_account.sql | 5 --- model/zuora/zuora_amendment.sql | 6 --- model/zuora/zuora_rate_plan.sql | 5 --- model/zuora/zuora_rate_plan_charge.sql | 8 ---- model/zuora/zuora_subscription.sql | 12 ------ ...subscriptions_w_charges_and_amendments.sql | 27 -------------- 18 files changed, 276 deletions(-) delete mode 100644 model/email/emails_denormalized.sql delete mode 100644 model/magento/order_items.sql delete mode 100644 model/magento/products.sql delete mode 100644 model/pardot/pardot_emails.sql delete mode 100644 model/pardot/pardot_tests.sql delete mode 100644 model/pardot/pardot_visitoractivity.sql delete mode 100644 model/pardot/pardot_visitoractivity_events_meta.sql delete mode 100644 model/pardot/pardot_visitoractivity_types_meta.sql delete mode 100644 model/segment/segment_track.sql delete mode 100644 model/snowplow/snowplow_events.sql delete mode 100644 model/trello/trello_card_location.sql delete mode 100644 model/trello/trello_model_tests.sql delete mode 100644 model/zuora/zuora_account.sql delete mode 100644 model/zuora/zuora_amendment.sql delete mode 100644 model/zuora/zuora_rate_plan.sql delete mode 100644 model/zuora/zuora_rate_plan_charge.sql delete mode 100644 model/zuora/zuora_subscription.sql delete mode 100644 model/zuora/zuora_subscriptions_w_charges_and_amendments.sql diff --git a/model/email/emails_denormalized.sql b/model/email/emails_denormalized.sql deleted file mode 100644 index a483de7..0000000 --- a/model/email/emails_denormalized.sql +++ /dev/null @@ -1,36 +0,0 @@ -with events as ( - - select * from {{env.schema}}.pardot_emails - -), - -sends as ( - - select "@user_id", "@timestamp", "@email_id" - from events - where "@event" = 'email sent' - -), - -opens as ( - - select "@user_id", "@timestamp", "@email_id" - from events - where "@event" = 'email opened' - -), clicks as ( - - select "@user_id", "@timestamp", "@email_id" - from events - where "@event" = 'email click' - -) - -select s."@user_id", s."@timestamp" as sent_timestamp, o."@timestamp" as opened_timestamp, c."@timestamp" as clicked_timestamp, - decode(o."@timestamp", null, 0, 1) as "opened?", - decode(c."@timestamp", null, 0, 1) as "clicked?", - row_number() over (partition by s."@user_id" order by s."@timestamp") as email_number -from sends s - left outer join opens o on s."@email_id" = o."@email_id" - left outer join clicks c on s."@email_id" = c."@email_id" -order by 1, 2 diff --git a/model/magento/order_items.sql b/model/magento/order_items.sql deleted file mode 100644 index 07d33d3..0000000 --- a/model/magento/order_items.sql +++ /dev/null @@ -1,3 +0,0 @@ -select * -FROM - sample_magento_database.sales_flat_order_item diff --git a/model/magento/products.sql b/model/magento/products.sql deleted file mode 100644 index 87468fb..0000000 --- a/model/magento/products.sql +++ /dev/null @@ -1,4 +0,0 @@ -select * -FROM - sample_magento_database.catalog_product_flat_1 - diff --git a/model/pardot/pardot_emails.sql b/model/pardot/pardot_emails.sql deleted file mode 100644 index b8308f1..0000000 --- a/model/pardot/pardot_emails.sql +++ /dev/null @@ -1,9 +0,0 @@ -/* -This model maps pardot data from the visitoractivity table to the email analysis interface. -It conforms to the basic email interface, not the extended email interface, because Pardot does not supply -data necessary to conform to the extended interface. -*/ - -select "@timestamp", "@event", "@user_id", email_id as "@email_id", details as "@subject" - from {{env.schema}}.pardot_visitoractivity -where "@event" in ('email sent', 'email opened', 'email click') diff --git a/model/pardot/pardot_tests.sql b/model/pardot/pardot_tests.sql deleted file mode 100644 index 20155df..0000000 --- a/model/pardot/pardot_tests.sql +++ /dev/null @@ -1,20 +0,0 @@ -select - 'visitoractivity_fresher_than_one_day' as name, - 'Most recent visitoractivity entry is no more than one day old' as description, - max("@timestamp"::timestamp) > current_date - '1 day'::interval as result -from {{env.schema}}.pardot_visitoractivity - - - - - - - -/* - -Other tests I want to do: -- make sure there are records from every day since the first day we see any records -- make sure all prospect ids from visitoractivity show up in prospects -- make sure there are no unmapped types - -*/ diff --git a/model/pardot/pardot_visitoractivity.sql b/model/pardot/pardot_visitoractivity.sql deleted file mode 100644 index f56509a..0000000 --- a/model/pardot/pardot_visitoractivity.sql +++ /dev/null @@ -1,15 +0,0 @@ ---this table has a bunch of types that really should be event actions but are very poorly formulated. ---the custom logic in this view is an attempt to fix that. ---not all of the various type / type_name combinations have been accounted for yet; I still need to determine exactly what some of them mean. -select - -- event_stream interface - va.created_at as "@timestamp", - e.event_name as "@event", - va.prospect_id as "@user_id", - va.* -from - olga_pardot.visitoractivity va - inner join {{env.schema}}.pardot_visitoractivity_events_meta e - on va."type" = e."type" and va.type_name = e.type_name - inner join {{env.schema}}.pardot_visitoractivity_types_meta t - on va."type" = t."type" diff --git a/model/pardot/pardot_visitoractivity_events_meta.sql b/model/pardot/pardot_visitoractivity_events_meta.sql deleted file mode 100644 index 2339b65..0000000 --- a/model/pardot/pardot_visitoractivity_events_meta.sql +++ /dev/null @@ -1,35 +0,0 @@ ---even with the type decoding that Pardot specifically provides, actually what is going on in a given event ---is somewhat ambiguous. this is an attempt to map type and type_name to a more event-based "event action" field ---which is always written in more standard action-oriented terms. -select 22 as "type", 'Chat Transcript' as type_name, 'chatted via olark' as event_name union all -select 21, 'Custom Redirect', 'clicked a custom redirect' union all -select 6, 'Email', 'email sent' union all -select 11, 'Email', 'email opened' union all -select 13, 'Email', 'email bounced' union all -select 14, 'Email', 'email reported spam' union all -select 1, 'Email Tracker', 'email click' union all -select 28, 'Event', 'registered for event' union all -select 29, 'Event', 'checked in at event' union all -select 2, 'File', 'viewed a file' union all -select 3, 'Form', 'submitted a form with an error' union all -select 2, 'Form', 'viewed a form' union all -select 4, 'Form', 'successfully submitted a form' union all -select 4, 'Form Handler', 'successfully submitted a form handler' union all -select 2, 'Landing Page', 'viewed a landing page' union all -select 4, 'Landing Page', 'successfully submitted the form on a landing page' union all -select 3, 'Landing Page', 'submitted the form on a landing page with an error' union all -select 2, 'Multivariate Landing Page', 'viewed multivariate landing page' union all -select 4, 'Multivariate Landing Page', 'successfully submitted multivariate landing page' union all -select 3, 'Multivariate Landing Page', 'submitted multivariate landing page with an error' union all -select 8, 'New Opportunity', 'opened opportunity' union all -select 19, 'Opportunity Associated', 'linked existing opportunity' union all -select 10, 'Opportunity Lost', 'lost opportunity' union all -select 9, 'Opportunity Won', 'won opportunity' union all -select 2, 'Page View', 'viewed highlighted page' union all -select 34, 'Video', 'watched 75% or more of video' union all -select 27, 'Video', 'watched video' union all -select 30, 'Video', 'converted from video call to action' union all -select 20, 'Visit', 'visited website' union all -select 25, 'Webinar', 'registered for webinar' union all -select 24, 'Webinar', 'attended webinar' union all -select 18, '', 'reopened opportunity' diff --git a/model/pardot/pardot_visitoractivity_types_meta.sql b/model/pardot/pardot_visitoractivity_types_meta.sql deleted file mode 100644 index 0e664bf..0000000 --- a/model/pardot/pardot_visitoractivity_types_meta.sql +++ /dev/null @@ -1,37 +0,0 @@ ---these literal values are pulled from pardot's api docs here: ---http://developer.pardot.com/kb/object-field-references/#visitor-activity ---they change periodically over time and this query will need to be correspondingly modified. -select 1 as type, 'Click' as type_decoded union all -select 2, 'View' union all -select 3, 'Error' union all -select 4, 'Success' union all -select 5, 'Session' union all -select 6, 'Sent' union all -select 7, 'Search' union all -select 8, 'New Opportunity' union all -select 9, 'Opportunity Won' union all -select 10, 'Opportunity Lost' union all -select 11, 'Open' union all -select 12, 'Unsubscribe Page' union all -select 13, 'Bounced' union all -select 14, 'Spam Complaint' union all -select 15, 'Email Preference Page' union all -select 16, 'Resubscribed' union all -select 17, 'Click (Third Party)' union all -select 18, 'Opportunity Reopened' union all -select 19, 'Opportunity Linked' union all -select 20, 'Visit' union all -select 21, 'Custom URL click' union all -select 22, 'Olark Chat' union all -select 23, 'Invited to Webinar' union all -select 24, 'Attended Webinar' union all -select 25, 'Registered for Webinar' union all -select 26, 'Social Post Click' union all -select 27, 'Video View' union all -select 28, 'Event Registered' union all -select 29, 'Event Checked In' union all -select 30, 'Video Conversion' union all -select 31, 'UserVoice Suggestion' union all -select 32, 'UserVoice Comment' union all -select 33, 'UserVoice Ticket' union all -select 34, 'Video Watched (>= 75% watched)' diff --git a/model/segment/segment_track.sql b/model/segment/segment_track.sql deleted file mode 100644 index b393fea..0000000 --- a/model/segment/segment_track.sql +++ /dev/null @@ -1,8 +0,0 @@ -select - "timestamp"::timestamp as "@timestamp", - "event" as "@event", - "userid" as "@user_id", - * - -from - segment.track diff --git a/model/snowplow/snowplow_events.sql b/model/snowplow/snowplow_events.sql deleted file mode 100644 index f438b35..0000000 --- a/model/snowplow/snowplow_events.sql +++ /dev/null @@ -1,7 +0,0 @@ -select - "collector_tstamp" as "@timestamp", - "event_name" as "@event", - "domain_userid" as "@user_id", - * -from - atomic.events diff --git a/model/trello/trello_card_location.sql b/model/trello/trello_card_location.sql deleted file mode 100644 index f6ebe45..0000000 --- a/model/trello/trello_card_location.sql +++ /dev/null @@ -1,18 +0,0 @@ -select - id, - idmembercreator, - date, - "type", - data__card__id, - data__card__name, - coalesce(data__list__id, - data__listafter__id, - lag(coalesce(data__list__id, data__listafter__id)) ignore nulls over (partition by data__card__id order by date)) as data__list__id, - coalesce(data__boardtarget__id, data__board__id) as data__board__id, - coalesce(data__card__closed, - lag(data__card__closed) ignore nulls over (partition by data__card__id order by date), - false) as data__card__closed -from trello_growth.trello_actions -where - data__card__id is not null - and "type" in ('createCard', 'updateCard', 'moveCardFromBoard', 'moveCardToBoard', 'commentCard') diff --git a/model/trello/trello_model_tests.sql b/model/trello/trello_model_tests.sql deleted file mode 100644 index 9f9840b..0000000 --- a/model/trello/trello_model_tests.sql +++ /dev/null @@ -1,21 +0,0 @@ -with null_boards_or_lists as -( - select id - from {{env.schema}}.trello_card_location - where - data__board__id is null - or data__list__id is null -) -select - 'no_null_boards_or_lists' as name, - 'All location entries have a non-null board and list id' as description, -count(*) = 0 as result -from null_boards_or_lists - -union all - -select - 'fresher_than_one_day', - 'Most recent entry is no more than one day old', -max(date::timestamp) > current_date - '1 day'::interval -from {{env.schema}}.trello_card_location diff --git a/model/zuora/zuora_account.sql b/model/zuora/zuora_account.sql deleted file mode 100644 index 233c201..0000000 --- a/model/zuora/zuora_account.sql +++ /dev/null @@ -1,5 +0,0 @@ -select - id as account_id, - accountnumber as account_number, - * -from zuora.zuora_account diff --git a/model/zuora/zuora_amendment.sql b/model/zuora/zuora_amendment.sql deleted file mode 100644 index 567b2e9..0000000 --- a/model/zuora/zuora_amendment.sql +++ /dev/null @@ -1,6 +0,0 @@ -select - id as amend_id, - subscriptionid as subscr_id, - effectivedate::timestamp as amend_start, - * -from zuora.zuora_amendment diff --git a/model/zuora/zuora_rate_plan.sql b/model/zuora/zuora_rate_plan.sql deleted file mode 100644 index e68af44..0000000 --- a/model/zuora/zuora_rate_plan.sql +++ /dev/null @@ -1,5 +0,0 @@ -select - id as rate_plan_id, - subscriptionid as subscr_id, - * -from zuora.zuora_rate_plan diff --git a/model/zuora/zuora_rate_plan_charge.sql b/model/zuora/zuora_rate_plan_charge.sql deleted file mode 100644 index eecf00d..0000000 --- a/model/zuora/zuora_rate_plan_charge.sql +++ /dev/null @@ -1,8 +0,0 @@ -select - rateplanid as rate_plan_id, - effectivestartdate::timestamp as rpc_start, - effectiveenddate::timestamp as rpc_end, - mrr as "@mrr", - islastsegment as rpc_last_segment, - * -from zuora.zuora_rate_plan_charge diff --git a/model/zuora/zuora_subscription.sql b/model/zuora/zuora_subscription.sql deleted file mode 100644 index ecae2de..0000000 --- a/model/zuora/zuora_subscription.sql +++ /dev/null @@ -1,12 +0,0 @@ -select - id as subscr_id, - status as subscr_status, - termtype as subscr_term_type, - accountid as account_id, - contracteffectivedate::timestamp as subscr_start, - subscriptionenddate::timestamp as subscr_end, - name as subscr_name, - "version#392c30e6081c24fb78ddf6d622de4f33"::integer - as subscr_version, - * -from zuora.zuora_subscription diff --git a/model/zuora/zuora_subscriptions_w_charges_and_amendments.sql b/model/zuora/zuora_subscriptions_w_charges_and_amendments.sql deleted file mode 100644 index 4da0efa..0000000 --- a/model/zuora/zuora_subscriptions_w_charges_and_amendments.sql +++ /dev/null @@ -1,27 +0,0 @@ --- get all subscriptions with possible ammendments for all accounts -with subscr_w_amendments as -( - select - account_number, acc.account_id, sub.subscr_id, - subscr_name, subscr_status, subscr_term_type, - subscr_start, subscr_end, subscr_version, amend_id, amend_start - from {{env.schema}}.zuora_account acc - inner join {{env.schema}}.zuora_subscription sub - on acc.account_id = sub.account_id - -- add ammendments - left outer join {{env.schema}}.zuora_amendment amend - on sub.subscr_id = amend.subscr_id -) - -select - account_number, account_id, sub.subscr_id, - subscr_name, subscr_status, subscr_term_type, - subscr_start, subscr_end, subscr_version, amend_id, amend_start, - rpc_start, rpc_end, rpc_last_segment, - min(subscr_start) over() as first_subscr, - "@mrr" as mrr -from subscr_w_amendments sub -inner join {{env.schema}}.zuora_rate_plan rp - on rp.subscr_id = sub.subscr_id -inner join {{env.schema}}.zuora_rate_plan_charge rpc - on rpc.rate_plan_id = rp.rate_plan_id From dda5bff5cd427a9ab6fec380bad050f3447af020 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 29 Mar 2016 15:46:29 -0400 Subject: [PATCH 58/88] dedent stripe code --- models/stripe/stripe_invoices.sql | 23 ++-- models/stripe/stripe_invoices_cleaned.sql | 22 ++-- models/stripe/stripe_invoices_transformed.sql | 116 +++++++++--------- models/stripe/stripe_plans.sql | 10 +- models/stripe/stripe_subscriptions.sql | 10 +- 5 files changed, 90 insertions(+), 91 deletions(-) diff --git a/models/stripe/stripe_invoices.sql b/models/stripe/stripe_invoices.sql index 14a5f74..1e3c89c 100644 --- a/models/stripe/stripe_invoices.sql +++ b/models/stripe/stripe_invoices.sql @@ -1,14 +1,13 @@ - select - customer, - date, - forgiven, - subscription as subscription_id, - paid, - total, - period_start, - period_end - from - demo_data.stripe_invoices - +select + customer, + date, + forgiven, + subscription as subscription_id, + paid, + total, + period_start, + period_end +from + demo_data.stripe_invoices diff --git a/models/stripe/stripe_invoices_cleaned.sql b/models/stripe/stripe_invoices_cleaned.sql index 74dfbdc..fa38136 100644 --- a/models/stripe/stripe_invoices_cleaned.sql +++ b/models/stripe/stripe_invoices_cleaned.sql @@ -1,14 +1,14 @@ - select - customer, - timestamp 'epoch' + date * interval '1 Second' as date, - forgiven, - subscription_id, - paid, - total, - timestamp 'epoch' + period_start * interval '1 Second' as period_start, - timestamp 'epoch' + period_end * interval '1 Second' as period_end - from - {{env.schema}}.stripe_invoices +select + customer, + timestamp 'epoch' + date * interval '1 Second' as date, + forgiven, + subscription_id, + paid, + total, + timestamp 'epoch' + period_start * interval '1 Second' as period_start, + timestamp 'epoch' + period_end * interval '1 Second' as period_end +from + {{env.schema}}.stripe_invoices diff --git a/models/stripe/stripe_invoices_transformed.sql b/models/stripe/stripe_invoices_transformed.sql index fd6b48f..2c824a1 100644 --- a/models/stripe/stripe_invoices_transformed.sql +++ b/models/stripe/stripe_invoices_transformed.sql @@ -1,61 +1,61 @@ - with invoices as ( - - select * - from {{env.schema}}.stripe_invoices_cleaned - where paid is true - and forgiven is false - - ), days as ( - - select (min(period_start) over () + row_number() over ())::date as date_day - from invoices - - ), months as ( - - select distinct date_trunc('month', date_day)::date as date_month - from days - where date_day <= current_date - - ), customers as ( - - select customer, min(period_start) as active_from, max(period_end) as active_to - from invoices - where period_start <= current_date - group by customer - - ), customer_dates as ( - - select m.date_month, c.customer - from months m - inner join customers c - on m.date_month >= date_trunc('month', c.active_from) - and m.date_month < date_trunc('month', c.active_to) - - ) - - select date_month, d.customer, period_start, period_end, - "interval" as period, - case "interval" - when 'yearly' - then coalesce(i.total, 0)::float / 12 / 100 - else - coalesce(i.total, 0)::float / 100 - end as total, - case min(date_month) over(partition by d.customer) - when date_month then 1 - else 0 - end as first_payment, - case max(date_month) over(partition by d.customer) - when date_month then 1 - else 0 - end as last_payment - from customer_dates d - left outer join invoices i - on d.date_month >= date_trunc('month', i.period_start) - and d.date_month < date_trunc('month', i.period_end) - and d.customer = i.customer - left outer join {{env.schema}}.stripe_subscriptions s on i.subscription_id = s.id - left outer join {{env.schema}}.stripe_plans p on s.plan_id = p.id +with invoices as ( + + select * + from {{env.schema}}.stripe_invoices_cleaned + where paid is true + and forgiven is false + +), days as ( + + select (min(period_start) over () + row_number() over ())::date as date_day + from invoices + +), months as ( + + select distinct date_trunc('month', date_day)::date as date_month + from days + where date_day <= current_date + +), customers as ( + + select customer, min(period_start) as active_from, max(period_end) as active_to + from invoices + where period_start <= current_date + group by customer + +), customer_dates as ( + + select m.date_month, c.customer + from months m + inner join customers c + on m.date_month >= date_trunc('month', c.active_from) + and m.date_month < date_trunc('month', c.active_to) + +) + +select date_month, d.customer, period_start, period_end, + "interval" as period, + case "interval" + when 'yearly' + then coalesce(i.total, 0)::float / 12 / 100 + else + coalesce(i.total, 0)::float / 100 + end as total, + case min(date_month) over(partition by d.customer) + when date_month then 1 + else 0 + end as first_payment, + case max(date_month) over(partition by d.customer) + when date_month then 1 + else 0 + end as last_payment +from customer_dates d + left outer join invoices i + on d.date_month >= date_trunc('month', i.period_start) + and d.date_month < date_trunc('month', i.period_end) + and d.customer = i.customer + left outer join {{env.schema}}.stripe_subscriptions s on i.subscription_id = s.id + left outer join {{env.schema}}.stripe_plans p on s.plan_id = p.id diff --git a/models/stripe/stripe_plans.sql b/models/stripe/stripe_plans.sql index 81a39b9..6600368 100644 --- a/models/stripe/stripe_plans.sql +++ b/models/stripe/stripe_plans.sql @@ -1,8 +1,8 @@ - select - id, - interval - from - demo_data.stripe_plans +select + id, + interval +from + demo_data.stripe_plans diff --git a/models/stripe/stripe_subscriptions.sql b/models/stripe/stripe_subscriptions.sql index bb1fe44..ce94ba0 100644 --- a/models/stripe/stripe_subscriptions.sql +++ b/models/stripe/stripe_subscriptions.sql @@ -1,8 +1,8 @@ - select - id, - plan__id as plan_id - from - demo_data.stripe_subscriptions +select + id, + plan__id as plan_id +from + demo_data.stripe_subscriptions From ded33b260912113a314d95667d9d7004e94b1840 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 29 Mar 2016 15:50:50 -0400 Subject: [PATCH 59/88] specify model and table_or_view in config --- dbt_project.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbt_project.yml b/dbt_project.yml index ed84403..1cd6ffc 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -3,3 +3,6 @@ source-paths: ["models"] target-path: "target" clean-targets: ["target"] + +models: ["*"] +table_or_view: 'table' From d97835a7cb74909b743a222ae685a7b45b974049 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 29 Mar 2016 15:51:29 -0400 Subject: [PATCH 60/88] add back in test path --- dbt_project.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt_project.yml b/dbt_project.yml index 1cd6ffc..f8f13ea 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -3,6 +3,7 @@ source-paths: ["models"] target-path: "target" clean-targets: ["target"] +test-paths: ["test"] models: ["*"] table_or_view: 'table' From a005f2cc7ae62505ef6534afc4d84c0ec320aed5 Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Wed, 30 Mar 2016 15:36:33 -0400 Subject: [PATCH 61/88] mixpanel v1 --- analysis/mixpanel/mixpanel_funnel.sql | 75 ++++++++++++++++++++ analysis/mixpanel/mixpanel_timeseries.sql | 12 ++++ models/mixpanel/model.sql | 84 +++++++++++++++++++++++ 3 files changed, 171 insertions(+) create mode 100644 analysis/mixpanel/mixpanel_funnel.sql create mode 100644 analysis/mixpanel/mixpanel_timeseries.sql create mode 100644 models/mixpanel/model.sql diff --git a/analysis/mixpanel/mixpanel_funnel.sql b/analysis/mixpanel/mixpanel_funnel.sql new file mode 100644 index 0000000..97955d5 --- /dev/null +++ b/analysis/mixpanel/mixpanel_funnel.sql @@ -0,0 +1,75 @@ +with +source as ( + select * from ac_yevgeniy.mixpanel_export -- change this view for your analysis +), + +step_1 as ( + select min(ac_timestamp) as ac_timestamp, ac_user_id + from source + where ac_event = 'event_1' -- filter by whichever columns you need + group by ac_user_id +), + +-- add more steps as you need. If you do add more steps, make sure to add a join below +step_2 as ( + select min(ac_timestamp) as ac_timestamp, ac_user_id + from source + where ac_event = 'event_2' + group by ac_user_id +), + +step_3 as ( + select min(ac_timestamp) as ac_timestamp, ac_user_id + from source + where ac_event = 'event_3' + group by ac_user_id +), + +step_4 as ( + select min(ac_timestamp) as ac_timestamp, ac_user_id + from source + where ac_event = 'event_4' + group by ac_user_id +), + +step_5 as ( + select min(ac_timestamp) as ac_timestamp, ac_user_id + from source + where ac_event = 'event_5' + group by ac_user_id +), + +temp_funnel as ( + + select 1 as funnel_idx, 'step_1' as funnel_step, count(distinct ac_user_id) as num_current_step + from step_1 + union + select 2 as funnel_idx, 'step_2' as funnel_step, count(distinct ac_user_id) as num_current_step + from step_2 + union + select 3 as funnel_idx, 'step_3' as funnel_step, count(distinct ac_user_id) as num_current_step + from step_3 + union + select 4 as funnel_idx, 'step_4' as funnel_step, count(distinct ac_user_id) as num_current_step + from step_4 + union + select 5 as funnel_idx, 'step_5' as funnel_step, count(distinct ac_user_id) as num_current_step + from step_5 +) + + +select + funnel_step, num_current_step as funnel_count, + round(100.0*num_current_step/num_previous_step, 2) as pcnt_previous, + round(100.0*num_current_step/num_first_step, 2) as pcnt_overall +from +( + select + funnel_idx, funnel_step, num_current_step, + -- lag the funnel numbers to compute conversion ratios + lag(num_current_step) over(order by funnel_idx) as num_previous_step, + -- get the first step number to compute the overall conversion ratio + max(num_current_step) over() as num_first_step + from temp_funnel +) +order by funnel_idx diff --git a/analysis/mixpanel/mixpanel_timeseries.sql b/analysis/mixpanel/mixpanel_timeseries.sql new file mode 100644 index 0000000..799af8a --- /dev/null +++ b/analysis/mixpanel/mixpanel_timeseries.sql @@ -0,0 +1,12 @@ +select + -- use second, minute, hour, day, week, month, quarter, etc + date_trunc('day', ac_timestamp) as period, count(*) period_count +from ac_yevgeniy.mixpanel_export +/* +--select the time horizon and specific events here +where + --ac_timestamp > getdate() - interval '1 week' + --and ac_event = 'event_1' +*/ +group by period +order by period asc diff --git a/models/mixpanel/model.sql b/models/mixpanel/model.sql new file mode 100644 index 0000000..89f5c8c --- /dev/null +++ b/models/mixpanel/model.sql @@ -0,0 +1,84 @@ +--create or replace view {{env.schema}}.mixpanel_export as +create or replace view ac_yevgeniy.mixpanel_export as +( + select + (timestamp 'epoch' + time * interval '1 Second') + as ac_timestamp, + event as ac_event, + distinct_id as ac_user_id, + * + from demo_data.mixpanel_export +); + + + + + +--create or replace view {{env.schema}}.mixpanel_export_with_sessions as ( +create or replace view ac_yevgeniy.mixpanel_export_with_sessions as +( + -- a new session is defined after 30 minutes of inactivity + with new_sessions as + ( + select + case + when extract(epoch from ac_timestamp) - lag(extract(epoch from ac_timestamp)) + over (partition by ac_user_id order by ac_timestamp) >= 30 * 60 + then 1 + else 0 + end as new_session, * + from ac_yevgeniy.mixpanel_export + ) + + -- make sure the first sessions is marked 1 and not 0 + select sum(new_session) + over (partition by ac_user_id order by ac_timestamp rows unbounded preceding) + 1 as session_idx, * + from new_sessions +); + + + + + +--create or replace view {{env.schema}}.mixpanel_cohort_data as ( +create or replace view ac_yevgeniy.mixpanel_cohort_data as +( + with + cohort_dates as + ( + select + ac_user_id, first_event_date, + -- get a cohort date for each user. Can use day, week, month, qtr, year + date_trunc('week', first_event_date)::date as cohort_date + from + ( + select ac_user_id, min(ac_timestamp) as first_event_date + from ac_yevgeniy.mixpanel_export + where + -- specifiy the cohort criterion, say based on event_1 being the user's first event + event = 'event_1' + group by ac_user_id + ) + ), + + second_event_dates as + ( + -- get a second event date + select ac_user_id, min(ac_timestamp) as second_event_date + from ac_yevgeniy.mixpanel_export + where + -- specify a second event of interest, say event_3 is a signup + event = 'event_3' + group by ac_user_id + ) + + -- get a table of cohort dates and first and second event dates + select cohort_date, first_event_date, second_event_date + from cohort_dates + left outer join second_event_dates + on cohort_dates.ac_user_id = second_event_dates.ac_user_id +); + + + + From df8ece5a87829a236deadf64afa82488d0bc696f Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Wed, 30 Mar 2016 15:39:19 -0400 Subject: [PATCH 62/88] mixpanel v1 --- analysis/mixpanel/mixpanel_funnel.sql | 2 +- analysis/mixpanel/mixpanel_timeseries.sql | 2 +- models/mixpanel/model.sql | 16 +++++++--------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/analysis/mixpanel/mixpanel_funnel.sql b/analysis/mixpanel/mixpanel_funnel.sql index 97955d5..b4a2c4d 100644 --- a/analysis/mixpanel/mixpanel_funnel.sql +++ b/analysis/mixpanel/mixpanel_funnel.sql @@ -1,6 +1,6 @@ with source as ( - select * from ac_yevgeniy.mixpanel_export -- change this view for your analysis + select * from {{env.schema}}.mixpanel_export -- change this view for your analysis ), step_1 as ( diff --git a/analysis/mixpanel/mixpanel_timeseries.sql b/analysis/mixpanel/mixpanel_timeseries.sql index 799af8a..efa3e6c 100644 --- a/analysis/mixpanel/mixpanel_timeseries.sql +++ b/analysis/mixpanel/mixpanel_timeseries.sql @@ -1,7 +1,7 @@ select -- use second, minute, hour, day, week, month, quarter, etc date_trunc('day', ac_timestamp) as period, count(*) period_count -from ac_yevgeniy.mixpanel_export +from {{env.schema}}.mixpanel_export /* --select the time horizon and specific events here where diff --git a/models/mixpanel/model.sql b/models/mixpanel/model.sql index 89f5c8c..06d8fda 100644 --- a/models/mixpanel/model.sql +++ b/models/mixpanel/model.sql @@ -1,5 +1,4 @@ ---create or replace view {{env.schema}}.mixpanel_export as -create or replace view ac_yevgeniy.mixpanel_export as +create or replace view {{env.schema}}.mixpanel_export as ( select (timestamp 'epoch' + time * interval '1 Second') @@ -14,8 +13,7 @@ create or replace view ac_yevgeniy.mixpanel_export as ---create or replace view {{env.schema}}.mixpanel_export_with_sessions as ( -create or replace view ac_yevgeniy.mixpanel_export_with_sessions as +create or replace view {{env.schema}}.mixpanel_export_with_sessions as ( -- a new session is defined after 30 minutes of inactivity with new_sessions as @@ -27,7 +25,7 @@ create or replace view ac_yevgeniy.mixpanel_export_with_sessions as then 1 else 0 end as new_session, * - from ac_yevgeniy.mixpanel_export + from {{env.schema}}.mixpanel_export ) -- make sure the first sessions is marked 1 and not 0 @@ -40,8 +38,8 @@ create or replace view ac_yevgeniy.mixpanel_export_with_sessions as ---create or replace view {{env.schema}}.mixpanel_cohort_data as ( -create or replace view ac_yevgeniy.mixpanel_cohort_data as + +create or replace view {{env.schema}}.mixpanel_cohort_data as ( with cohort_dates as @@ -53,7 +51,7 @@ create or replace view ac_yevgeniy.mixpanel_cohort_data as from ( select ac_user_id, min(ac_timestamp) as first_event_date - from ac_yevgeniy.mixpanel_export + from {{env.schema}}.mixpanel_export where -- specifiy the cohort criterion, say based on event_1 being the user's first event event = 'event_1' @@ -65,7 +63,7 @@ create or replace view ac_yevgeniy.mixpanel_cohort_data as ( -- get a second event date select ac_user_id, min(ac_timestamp) as second_event_date - from ac_yevgeniy.mixpanel_export + from {{env.schema}}.mixpanel_export where -- specify a second event of interest, say event_3 is a signup event = 'event_3' From 0c75a6a83a5c897470b4a96632c163724bbcdd36 Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Thu, 31 Mar 2016 16:50:27 -0400 Subject: [PATCH 63/88] added retention analysis --- analysis/mixpanel/mixpanel_retention.sql | 84 ++++++++++++++++++++++++ models/mixpanel/model.sql | 10 +++ 2 files changed, 94 insertions(+) create mode 100644 analysis/mixpanel/mixpanel_retention.sql diff --git a/analysis/mixpanel/mixpanel_retention.sql b/analysis/mixpanel/mixpanel_retention.sql new file mode 100644 index 0000000..2eb3f22 --- /dev/null +++ b/analysis/mixpanel/mixpanel_retention.sql @@ -0,0 +1,84 @@ +with events as +( + -- can change 'week' to 'day', 'month', 'qtr', etc. + -- Make sure to make corresponding everywhere below + select date_trunc('week', ac_timestamp)::date as ac_date, * + from {{env.schema}}.mixpanel_export +), + +users as +( + select ac_user_id, min(date_trunc('week', ac_user_created)::date) as ac_user_created + from {{env.schema}}.mixpanel_engage + group by ac_user_id +), + +overall_retention as +( + select *, 100.0*retained_users/active_users as overall_retention + from + ( + select + events.ac_date, + count(distinct events.ac_user_id) as active_users, + count(distinct future_events.ac_user_id) as retained_users + from events + -- join future events + left outer join events as future_events + on events.ac_user_id = future_events.ac_user_id + and events.ac_date = future_events.ac_date - interval '1 week' + group by events.ac_date + ) +), + +new_retention as +( + select *, 100.0*retained_users/active_users as new_retention + from + ( + select + events.ac_date, + count(distinct events.ac_user_id) as active_users, + count(distinct future_events.ac_user_id) as retained_users + from events + left outer join events as future_events + on events.ac_user_id = future_events.ac_user_id + and events.ac_date = future_events.ac_date - interval '1 week' + inner join users + on events.ac_user_id = users.ac_user_id + and events.ac_date = users.ac_user_created + group by events.ac_date + ) +), + +old_retention as +( + select *, 100.0*retained_users/active_users as old_retention + from + ( + select + events.ac_date, + count(distinct events.ac_user_id) as active_users, + count(distinct future_events.ac_user_id) as retained_users + from events + left outer join events as future_events + on events.ac_user_id = future_events.ac_user_id + and events.ac_date = future_events.ac_date - interval '1 week' + inner join users + on events.ac_user_id = users.ac_user_id + and events.ac_date != users.ac_user_created + group by events.ac_date + ) +) + +select + overall_retention.ac_date, overall_retention as overall_retention_pcnt, + new_retention as new_retention_pcnt, old_retention as old_retention_pcnt +from overall_retention +left outer join new_retention + on overall_retention.ac_date = new_retention.ac_date +left outer join old_retention + on overall_retention.ac_date = old_retention.ac_date + + + diff --git a/models/mixpanel/model.sql b/models/mixpanel/model.sql index 06d8fda..2677c9f 100644 --- a/models/mixpanel/model.sql +++ b/models/mixpanel/model.sql @@ -1,3 +1,13 @@ +create or replace view {{env.schema}}.mixpanel_engage as +( + select + mp_reserved_created as ac_user_created, + mp_reserved_distinct_id as ac_user_id, + * + from demo_data.mixpanel_engage +); + + create or replace view {{env.schema}}.mixpanel_export as ( select From 5128e0389997d9246c8714460a82ccc4d1214d63 Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Fri, 1 Apr 2016 12:01:03 -0400 Subject: [PATCH 64/88] * changes --- analysis/mixpanel/mixpanel_funnel.sql | 40 +++++++-------- analysis/mixpanel/mixpanel_retention.sql | 62 +++++++++++------------ analysis/mixpanel/mixpanel_timeseries.sql | 4 +- models/mixpanel/model.sql | 30 +++++------ 4 files changed, 66 insertions(+), 70 deletions(-) diff --git a/analysis/mixpanel/mixpanel_funnel.sql b/analysis/mixpanel/mixpanel_funnel.sql index b4a2c4d..c689335 100644 --- a/analysis/mixpanel/mixpanel_funnel.sql +++ b/analysis/mixpanel/mixpanel_funnel.sql @@ -4,56 +4,56 @@ source as ( ), step_1 as ( - select min(ac_timestamp) as ac_timestamp, ac_user_id + select min(event_date) as event_date, user_id from source - where ac_event = 'event_1' -- filter by whichever columns you need - group by ac_user_id + where event = 'event_1' -- filter by whichever columns you need + group by user_id ), -- add more steps as you need. If you do add more steps, make sure to add a join below step_2 as ( - select min(ac_timestamp) as ac_timestamp, ac_user_id + select min(event_date) as event_date, user_id from source - where ac_event = 'event_2' - group by ac_user_id + where event = 'event_2' + group by user_id ), step_3 as ( - select min(ac_timestamp) as ac_timestamp, ac_user_id + select min(event_date) as event_date, user_id from source - where ac_event = 'event_3' - group by ac_user_id + where event = 'event_3' + group by user_id ), step_4 as ( - select min(ac_timestamp) as ac_timestamp, ac_user_id + select min(event_date) as event_date, user_id from source - where ac_event = 'event_4' - group by ac_user_id + where event = 'event_4' + group by user_id ), step_5 as ( - select min(ac_timestamp) as ac_timestamp, ac_user_id + select min(event_date) as event_date, user_id from source - where ac_event = 'event_5' - group by ac_user_id + where event = 'event_5' + group by user_id ), temp_funnel as ( - select 1 as funnel_idx, 'step_1' as funnel_step, count(distinct ac_user_id) as num_current_step + select 1 as funnel_idx, 'step_1' as funnel_step, count(distinct user_id) as num_current_step from step_1 union - select 2 as funnel_idx, 'step_2' as funnel_step, count(distinct ac_user_id) as num_current_step + select 2 as funnel_idx, 'step_2' as funnel_step, count(distinct user_id) as num_current_step from step_2 union - select 3 as funnel_idx, 'step_3' as funnel_step, count(distinct ac_user_id) as num_current_step + select 3 as funnel_idx, 'step_3' as funnel_step, count(distinct user_id) as num_current_step from step_3 union - select 4 as funnel_idx, 'step_4' as funnel_step, count(distinct ac_user_id) as num_current_step + select 4 as funnel_idx, 'step_4' as funnel_step, count(distinct user_id) as num_current_step from step_4 union - select 5 as funnel_idx, 'step_5' as funnel_step, count(distinct ac_user_id) as num_current_step + select 5 as funnel_idx, 'step_5' as funnel_step, count(distinct user_id) as num_current_step from step_5 ) diff --git a/analysis/mixpanel/mixpanel_retention.sql b/analysis/mixpanel/mixpanel_retention.sql index 2eb3f22..052d9dd 100644 --- a/analysis/mixpanel/mixpanel_retention.sql +++ b/analysis/mixpanel/mixpanel_retention.sql @@ -2,83 +2,83 @@ with events as ( -- can change 'week' to 'day', 'month', 'qtr', etc. -- Make sure to make corresponding everywhere below - select date_trunc('week', ac_timestamp)::date as ac_date, * + select date_trunc('week', event_date)::date as event_date, event, user_id from {{env.schema}}.mixpanel_export ), users as ( - select ac_user_id, min(date_trunc('week', ac_user_created)::date) as ac_user_created + select user_id, min(date_trunc('week', user_created)::date) as user_created from {{env.schema}}.mixpanel_engage - group by ac_user_id + group by user_id ), overall_retention as ( - select *, 100.0*retained_users/active_users as overall_retention + select *, retained_users::float/active_users as overall_retention from ( select - events.ac_date, - count(distinct events.ac_user_id) as active_users, - count(distinct future_events.ac_user_id) as retained_users + events.event_date, + count(distinct events.user_id) as active_users, + count(distinct future_events.user_id) as retained_users from events -- join future events left outer join events as future_events - on events.ac_user_id = future_events.ac_user_id - and events.ac_date = future_events.ac_date - interval '1 week' - group by events.ac_date + on events.user_id = future_events.user_id + and events.event_date = future_events.event_date - interval '1 week' + group by events.event_date ) ), new_retention as ( - select *, 100.0*retained_users/active_users as new_retention + select *, retained_users::float/active_users as new_retention from ( select - events.ac_date, - count(distinct events.ac_user_id) as active_users, - count(distinct future_events.ac_user_id) as retained_users + events.event_date, + count(distinct events.user_id) as active_users, + count(distinct future_events.user_id) as retained_users from events left outer join events as future_events - on events.ac_user_id = future_events.ac_user_id - and events.ac_date = future_events.ac_date - interval '1 week' + on events.user_id = future_events.user_id + and events.event_date = future_events.event_date - interval '1 week' inner join users - on events.ac_user_id = users.ac_user_id - and events.ac_date = users.ac_user_created - group by events.ac_date + on events.user_id = users.user_id + and events.event_date = users.user_created + group by events.event_date ) ), old_retention as ( - select *, 100.0*retained_users/active_users as old_retention + select *, retained_users::float/active_users as old_retention from ( select - events.ac_date, - count(distinct events.ac_user_id) as active_users, - count(distinct future_events.ac_user_id) as retained_users + events.event_date, + count(distinct events.user_id) as active_users, + count(distinct future_events.user_id) as retained_users from events left outer join events as future_events - on events.ac_user_id = future_events.ac_user_id - and events.ac_date = future_events.ac_date - interval '1 week' + on events.user_id = future_events.user_id + and events.event_date = future_events.event_date - interval '1 week' inner join users - on events.ac_user_id = users.ac_user_id - and events.ac_date != users.ac_user_created - group by events.ac_date + on events.user_id = users.user_id + and events.event_date != users.user_created + group by events.event_date ) ) select - overall_retention.ac_date, overall_retention as overall_retention_pcnt, + overall_retention.event_date, overall_retention as overall_retention_pcnt, new_retention as new_retention_pcnt, old_retention as old_retention_pcnt from overall_retention left outer join new_retention - on overall_retention.ac_date = new_retention.ac_date + on overall_retention.event_date = new_retention.event_date left outer join old_retention - on overall_retention.ac_date = old_retention.ac_date + on overall_retention.event_date = old_retention.event_date diff --git a/analysis/mixpanel/mixpanel_timeseries.sql b/analysis/mixpanel/mixpanel_timeseries.sql index efa3e6c..bb22a52 100644 --- a/analysis/mixpanel/mixpanel_timeseries.sql +++ b/analysis/mixpanel/mixpanel_timeseries.sql @@ -1,11 +1,11 @@ select -- use second, minute, hour, day, week, month, quarter, etc - date_trunc('day', ac_timestamp) as period, count(*) period_count + date_trunc('day', event_date) as period, count(*) period_count from {{env.schema}}.mixpanel_export /* --select the time horizon and specific events here where - --ac_timestamp > getdate() - interval '1 week' + --event_date > getdate() - interval '1 week' --and ac_event = 'event_1' */ group by period diff --git a/models/mixpanel/model.sql b/models/mixpanel/model.sql index 2677c9f..407accb 100644 --- a/models/mixpanel/model.sql +++ b/models/mixpanel/model.sql @@ -1,9 +1,8 @@ create or replace view {{env.schema}}.mixpanel_engage as ( select - mp_reserved_created as ac_user_created, - mp_reserved_distinct_id as ac_user_id, - * + mp_reserved_created as user_created, + mp_reserved_distinct_id as user_id from demo_data.mixpanel_engage ); @@ -11,11 +10,8 @@ create or replace view {{env.schema}}.mixpanel_engage as create or replace view {{env.schema}}.mixpanel_export as ( select - (timestamp 'epoch' + time * interval '1 Second') - as ac_timestamp, - event as ac_event, - distinct_id as ac_user_id, - * + (timestamp 'epoch' + time * interval '1 Second') as event_date, + event, distinct_id as user_id from demo_data.mixpanel_export ); @@ -30,8 +26,8 @@ create or replace view {{env.schema}}.mixpanel_export_with_sessions as ( select case - when extract(epoch from ac_timestamp) - lag(extract(epoch from ac_timestamp)) - over (partition by ac_user_id order by ac_timestamp) >= 30 * 60 + when extract(epoch from event_date) - lag(extract(epoch from event_date)) + over (partition by user_id order by event_date) >= 30 * 60 then 1 else 0 end as new_session, * @@ -40,7 +36,7 @@ create or replace view {{env.schema}}.mixpanel_export_with_sessions as -- make sure the first sessions is marked 1 and not 0 select sum(new_session) - over (partition by ac_user_id order by ac_timestamp rows unbounded preceding) + 1 as session_idx, * + over (partition by user_id order by event_date rows unbounded preceding) + 1 as session_idx, * from new_sessions ); @@ -55,36 +51,36 @@ create or replace view {{env.schema}}.mixpanel_cohort_data as cohort_dates as ( select - ac_user_id, first_event_date, + user_id, first_event_date, -- get a cohort date for each user. Can use day, week, month, qtr, year date_trunc('week', first_event_date)::date as cohort_date from ( - select ac_user_id, min(ac_timestamp) as first_event_date + select user_id, min(event_date) as first_event_date from {{env.schema}}.mixpanel_export where -- specifiy the cohort criterion, say based on event_1 being the user's first event event = 'event_1' - group by ac_user_id + group by user_id ) ), second_event_dates as ( -- get a second event date - select ac_user_id, min(ac_timestamp) as second_event_date + select user_id, min(event_date) as second_event_date from {{env.schema}}.mixpanel_export where -- specify a second event of interest, say event_3 is a signup event = 'event_3' - group by ac_user_id + group by user_id ) -- get a table of cohort dates and first and second event dates select cohort_date, first_event_date, second_event_date from cohort_dates left outer join second_event_dates - on cohort_dates.ac_user_id = second_event_dates.ac_user_id + on cohort_dates.user_id = second_event_dates.user_id ); From 1c3e80aa410feff07f6898abb15f8f521d8bc1ea Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Fri, 1 Apr 2016 17:31:59 -0400 Subject: [PATCH 65/88] added event cohorts --- analysis/mixpanel/mixpanel_event_cohorts.sql | 74 ++++++++++++++++++++ models/mixpanel/model.sql | 47 ------------- 2 files changed, 74 insertions(+), 47 deletions(-) create mode 100644 analysis/mixpanel/mixpanel_event_cohorts.sql diff --git a/analysis/mixpanel/mixpanel_event_cohorts.sql b/analysis/mixpanel/mixpanel_event_cohorts.sql new file mode 100644 index 0000000..963b4fb --- /dev/null +++ b/analysis/mixpanel/mixpanel_event_cohorts.sql @@ -0,0 +1,74 @@ +with +mixpanel_events as +( + select * + from {{env.schema}}.mixpanel_export +), + +first_event_data as +( + -- find all (user, date) records for the first event of interest + select + user_id, first_event_date + from + ( + -- pick the first occurence of the event of interest fo cohorting purposes + select user_id, min(event_date) as first_event_date + from mixpanel_events + where + -- specifiy the first event of interest + event = 'event_1' + group by user_id + ) +), + +second_event_data as ( + -- find all (user, date) records for the second event of interest + select user_id, event_date as second_event_date + from mixpanel_events + where + -- specify the second event of interest + event = 'visualize impactful channels' +), + +combined_event_data as +( + -- get the table of first and second event dates for each user + select first_event_data.user_id, first_event_date, second_event_date + from first_event_data + inner join second_event_data + on first_event_data.user_id = second_event_data.user_id +), + +cohort_sizes as +( + -- compute cohort sizes + select + date_trunc('week', first_event_date)::date as cohort_date, + count(distinct user_id) as cohort_size + from combined_event_data + group by date_trunc('week', first_event_date) +), + +cohort_data as ( + select cohorts.cohort_date, cohort_size, user_id, days_to_second_event + from + ( + select + user_id, date_trunc('week', first_event_date)::date as cohort_date, + datediff(day, first_event_date, second_event_date) as days_to_second_event + from combined_event_data + ) cohorts + inner join cohort_sizes + on cohorts.cohort_date = cohort_sizes.cohort_date +) + +select + cohort_date, cohort_size, days_to_second_event, count(distinct user_id) as users, + count(distinct user_id)::float/cohort_size as portion_of_users +from cohort_data +group by cohort_date, cohort_size, days_to_second_event +order by cohort_date, cohort_size, days_to_second_event + + + diff --git a/models/mixpanel/model.sql b/models/mixpanel/model.sql index 407accb..c45f428 100644 --- a/models/mixpanel/model.sql +++ b/models/mixpanel/model.sql @@ -17,8 +17,6 @@ create or replace view {{env.schema}}.mixpanel_export as - - create or replace view {{env.schema}}.mixpanel_export_with_sessions as ( -- a new session is defined after 30 minutes of inactivity @@ -41,48 +39,3 @@ create or replace view {{env.schema}}.mixpanel_export_with_sessions as ); - - - - -create or replace view {{env.schema}}.mixpanel_cohort_data as -( - with - cohort_dates as - ( - select - user_id, first_event_date, - -- get a cohort date for each user. Can use day, week, month, qtr, year - date_trunc('week', first_event_date)::date as cohort_date - from - ( - select user_id, min(event_date) as first_event_date - from {{env.schema}}.mixpanel_export - where - -- specifiy the cohort criterion, say based on event_1 being the user's first event - event = 'event_1' - group by user_id - ) - ), - - second_event_dates as - ( - -- get a second event date - select user_id, min(event_date) as second_event_date - from {{env.schema}}.mixpanel_export - where - -- specify a second event of interest, say event_3 is a signup - event = 'event_3' - group by user_id - ) - - -- get a table of cohort dates and first and second event dates - select cohort_date, first_event_date, second_event_date - from cohort_dates - left outer join second_event_dates - on cohort_dates.user_id = second_event_dates.user_id -); - - - - From 7e5e431808913acf4d092889b4880f0a77aa61f8 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Sun, 3 Apr 2016 14:57:14 -0400 Subject: [PATCH 66/88] remove old configs --- dbt_project.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index f8f13ea..0eab1f7 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -4,6 +4,3 @@ source-paths: ["models"] target-path: "target" clean-targets: ["target"] test-paths: ["test"] - -models: ["*"] -table_or_view: 'table' From b76a1448365f3495d3645b1f9508b98c7320e805 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Sun, 3 Apr 2016 17:23:10 -0400 Subject: [PATCH 67/88] update example configs --- dbt_project.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dbt_project.yml b/dbt_project.yml index 0eab1f7..1ab3d0d 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -4,3 +4,11 @@ source-paths: ["models"] target-path: "target" clean-targets: ["target"] test-paths: ["test"] + +model-defaults: + materialized: false + enabled: true + +#models: +# zuora: +# materialized: true From 598cd6b473089d84c7086d2751e88386be5164cd Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Tue, 5 Apr 2016 09:28:43 -0400 Subject: [PATCH 68/88] updates to readme based on current workflow --- README.md | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index bff1ffd..714f86a 100644 --- a/README.md +++ b/README.md @@ -2,16 +2,34 @@ A collection of data models and corresponding analysis for common data sets in SQL. These models are designed to be portable across organizations with minimal configuration. -### Contributing +### Design Principles + +This repository contains two primary types of objects: data models and data analyses. -##### About Models +##### Models - A model is a table or view built either on top of raw data or other models. Models are not transient; they are materialized in the database. -- Currently all models are views. Support for models-as-tables is anticipated at some point. -- Model files should go into `/models` and saved with a `.sql` extension. Folder structure within `/models` is for logical grouping only. -- All models are built to be compiled and run with [dbt](https://github.com/analyst-collective/dbt). Environment configuration should be supplied via `{{env}}` -- All models will need to be adapted to your environment so as to select data from the appropriate raw data tables and fields. Once this mapping has been completed, all subsequent analysis built on top of these models will function normally. +- Models are composed of a single SQL `select` statement. Any valid SQL can be used. As such, models can provide functionality such as data cleansing, data transformation, etc. +- All models are built to be compiled and run with [dbt](https://github.com/analyst-collective/dbt). +- Models can be configured in dbt to be materialized as either views or tables. +- Model files should go into `/models` and saved with a `.sql` extension. +- Each model should be stored in its own `.sql` file. The file name will become the name of the table or view in the database. +- Environment configuration should be supplied via `{{env}}`. +- Models should be designed to minimize the selection from raw data tables. This minimizes the amount of mapping end users of models will need to do when configuring them for their local environment. -##### About Analysis -- All analysis should be built on top of models. +##### Analysis +- Analyses are `.sql` files that can be executed within a database query tool. +- All analysis should be built on top of models, not raw data. - All named fields in a given analysis should be named within a given model. - Confining analysis in this way ensures portability of analysis across multiple environments. + + +### Contributing +All contributions to this repository must be for analytics on top of standardized datasets. The current process for contributing is to: +- fork this repo, +- build a test dataset, +- make and test changes, and +- submit a PR. + +PRs without accompanying datasets cannot be tested and therefore will not be accepted. We suggest you use [data-generator](https://github.com/analyst-collective/data-generator) to generate your test datasets. + +We do not believe that this is the ideal workflow to facilitate the Analyst Collective vision for open source analytics. In the future, we plan to extend dbt to be a package manager. Once this is accomplished, you can own your own analytics repositories and publish them to a common index that others can use. We will update the contribution guidelines here once this is accomplished. From 6c2a2c3a3cef2ae36f92b2ac92da17c44d98df72 Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Tue, 5 Apr 2016 16:04:20 -0400 Subject: [PATCH 69/88] updated mixpanel models --- analysis/mixpanel/mixpanel_event_cohorts.sql | 2 +- analysis/mixpanel/mixpanel_retention.sql | 84 ------------------- models/mixpanel/mixpanel_engage.sql | 4 + models/mixpanel/mixpanel_export.sql | 5 ++ .../mixpanel_export_with_sessions.sql | 17 ++++ models/mixpanel/model.sql | 41 --------- 6 files changed, 27 insertions(+), 126 deletions(-) delete mode 100644 analysis/mixpanel/mixpanel_retention.sql create mode 100644 models/mixpanel/mixpanel_engage.sql create mode 100644 models/mixpanel/mixpanel_export.sql create mode 100644 models/mixpanel/mixpanel_export_with_sessions.sql delete mode 100644 models/mixpanel/model.sql diff --git a/analysis/mixpanel/mixpanel_event_cohorts.sql b/analysis/mixpanel/mixpanel_event_cohorts.sql index 963b4fb..48e2446 100644 --- a/analysis/mixpanel/mixpanel_event_cohorts.sql +++ b/analysis/mixpanel/mixpanel_event_cohorts.sql @@ -12,7 +12,7 @@ first_event_data as user_id, first_event_date from ( - -- pick the first occurence of the event of interest fo cohorting purposes + -- pick the first occurence of the event of interest for cohorting purposes select user_id, min(event_date) as first_event_date from mixpanel_events where diff --git a/analysis/mixpanel/mixpanel_retention.sql b/analysis/mixpanel/mixpanel_retention.sql deleted file mode 100644 index 052d9dd..0000000 --- a/analysis/mixpanel/mixpanel_retention.sql +++ /dev/null @@ -1,84 +0,0 @@ -with events as -( - -- can change 'week' to 'day', 'month', 'qtr', etc. - -- Make sure to make corresponding everywhere below - select date_trunc('week', event_date)::date as event_date, event, user_id - from {{env.schema}}.mixpanel_export -), - -users as -( - select user_id, min(date_trunc('week', user_created)::date) as user_created - from {{env.schema}}.mixpanel_engage - group by user_id -), - -overall_retention as -( - select *, retained_users::float/active_users as overall_retention - from - ( - select - events.event_date, - count(distinct events.user_id) as active_users, - count(distinct future_events.user_id) as retained_users - from events - -- join future events - left outer join events as future_events - on events.user_id = future_events.user_id - and events.event_date = future_events.event_date - interval '1 week' - group by events.event_date - ) -), - -new_retention as -( - select *, retained_users::float/active_users as new_retention - from - ( - select - events.event_date, - count(distinct events.user_id) as active_users, - count(distinct future_events.user_id) as retained_users - from events - left outer join events as future_events - on events.user_id = future_events.user_id - and events.event_date = future_events.event_date - interval '1 week' - inner join users - on events.user_id = users.user_id - and events.event_date = users.user_created - group by events.event_date - ) -), - -old_retention as -( - select *, retained_users::float/active_users as old_retention - from - ( - select - events.event_date, - count(distinct events.user_id) as active_users, - count(distinct future_events.user_id) as retained_users - from events - left outer join events as future_events - on events.user_id = future_events.user_id - and events.event_date = future_events.event_date - interval '1 week' - inner join users - on events.user_id = users.user_id - and events.event_date != users.user_created - group by events.event_date - ) -) - -select - overall_retention.event_date, overall_retention as overall_retention_pcnt, - new_retention as new_retention_pcnt, old_retention as old_retention_pcnt -from overall_retention -left outer join new_retention - on overall_retention.event_date = new_retention.event_date -left outer join old_retention - on overall_retention.event_date = old_retention.event_date - - - diff --git a/models/mixpanel/mixpanel_engage.sql b/models/mixpanel/mixpanel_engage.sql new file mode 100644 index 0000000..495f269 --- /dev/null +++ b/models/mixpanel/mixpanel_engage.sql @@ -0,0 +1,4 @@ +select + mp_reserved_created as user_created, + mp_reserved_distinct_id as user_id +from demo_data.mixpanel_engage \ No newline at end of file diff --git a/models/mixpanel/mixpanel_export.sql b/models/mixpanel/mixpanel_export.sql new file mode 100644 index 0000000..b8430c1 --- /dev/null +++ b/models/mixpanel/mixpanel_export.sql @@ -0,0 +1,5 @@ +select + -- convert unix time to timestamp + (timestamp 'epoch' + time * interval '1 Second') as event_date, + event, distinct_id as user_id +from demo_data.mixpanel_export \ No newline at end of file diff --git a/models/mixpanel/mixpanel_export_with_sessions.sql b/models/mixpanel/mixpanel_export_with_sessions.sql new file mode 100644 index 0000000..52ca469 --- /dev/null +++ b/models/mixpanel/mixpanel_export_with_sessions.sql @@ -0,0 +1,17 @@ +-- a new session is defined after 30 minutes of inactivity +with new_sessions as +( + select + case + when extract(epoch from event_date) - lag(extract(epoch from event_date)) + over (partition by user_id order by event_date) >= 30 * 60 + then 1 + else 0 + end as new_session, * + from {{env.schema}}.mixpanel_export +) + +-- make sure the first sessions is marked 1 and not 0 +select sum(new_session) + over (partition by user_id order by event_date rows unbounded preceding) + 1 as session_idx, * +from new_sessions \ No newline at end of file diff --git a/models/mixpanel/model.sql b/models/mixpanel/model.sql deleted file mode 100644 index c45f428..0000000 --- a/models/mixpanel/model.sql +++ /dev/null @@ -1,41 +0,0 @@ -create or replace view {{env.schema}}.mixpanel_engage as -( - select - mp_reserved_created as user_created, - mp_reserved_distinct_id as user_id - from demo_data.mixpanel_engage -); - - -create or replace view {{env.schema}}.mixpanel_export as -( - select - (timestamp 'epoch' + time * interval '1 Second') as event_date, - event, distinct_id as user_id - from demo_data.mixpanel_export -); - - - -create or replace view {{env.schema}}.mixpanel_export_with_sessions as -( - -- a new session is defined after 30 minutes of inactivity - with new_sessions as - ( - select - case - when extract(epoch from event_date) - lag(extract(epoch from event_date)) - over (partition by user_id order by event_date) >= 30 * 60 - then 1 - else 0 - end as new_session, * - from {{env.schema}}.mixpanel_export - ) - - -- make sure the first sessions is marked 1 and not 0 - select sum(new_session) - over (partition by user_id order by event_date rows unbounded preceding) + 1 as session_idx, * - from new_sessions -); - - From 305e67b799f58e78d8f3728e1530d3f094e9ebb7 Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Wed, 6 Apr 2016 14:27:58 -0400 Subject: [PATCH 70/88] mailchimp v1 --- .../mailchimp/mailchimp_email_timeseries.sql | 14 +++++ models/mailchimp/mailchimp_campaigns.sql | 3 ++ models/mailchimp/mailchimp_email_actions.sql | 3 ++ .../mailchimp_emails_denormalized.sql | 53 +++++++++++++++++++ models/mailchimp/mailchimp_lists.sql | 3 ++ models/mailchimp/mailchimp_members.sql | 3 ++ models/mailchimp/mailchimp_sent_to.sql | 7 +++ 7 files changed, 86 insertions(+) create mode 100644 analysis/mailchimp/mailchimp_email_timeseries.sql create mode 100644 models/mailchimp/mailchimp_campaigns.sql create mode 100644 models/mailchimp/mailchimp_email_actions.sql create mode 100644 models/mailchimp/mailchimp_emails_denormalized.sql create mode 100644 models/mailchimp/mailchimp_lists.sql create mode 100644 models/mailchimp/mailchimp_members.sql create mode 100644 models/mailchimp/mailchimp_sent_to.sql diff --git a/analysis/mailchimp/mailchimp_email_timeseries.sql b/analysis/mailchimp/mailchimp_email_timeseries.sql new file mode 100644 index 0000000..aa877f2 --- /dev/null +++ b/analysis/mailchimp/mailchimp_email_timeseries.sql @@ -0,0 +1,14 @@ +with events as ( + select * from {{env.schema}}.mailchimp_emails_denormalized +) + +select + date_trunc('month', sent_date) as mnth, + count(*) as sends, + sum(opened) as opens, + sum(clicked) as clicks, + avg(opened::float) as open_rate, + avg(clicked::float) as click_rate +from events +group by 1 +order by 1 \ No newline at end of file diff --git a/models/mailchimp/mailchimp_campaigns.sql b/models/mailchimp/mailchimp_campaigns.sql new file mode 100644 index 0000000..c246487 --- /dev/null +++ b/models/mailchimp/mailchimp_campaigns.sql @@ -0,0 +1,3 @@ +select distinct + recipients__list_id as list_id, id as campaign_id, send_time as sent_date +from demo_data.mailchimp_campaigns \ No newline at end of file diff --git a/models/mailchimp/mailchimp_email_actions.sql b/models/mailchimp/mailchimp_email_actions.sql new file mode 100644 index 0000000..e322893 --- /dev/null +++ b/models/mailchimp/mailchimp_email_actions.sql @@ -0,0 +1,3 @@ +select + campaign_id, email_id, action, "timestamp" as action_date +from demo_data.mailchimp_email_activity__activity \ No newline at end of file diff --git a/models/mailchimp/mailchimp_emails_denormalized.sql b/models/mailchimp/mailchimp_emails_denormalized.sql new file mode 100644 index 0000000..525bf07 --- /dev/null +++ b/models/mailchimp/mailchimp_emails_denormalized.sql @@ -0,0 +1,53 @@ +with email_actions as ( + select * + from {{env.schema}}.mailchimp_email_actions +), + +sends as ( + select campaign_id, email_id, sent_date + from {{env.schema}}.mailchimp_sent_to +), + +bounces as +( + select campaign_id, email_id, min(action_date) as bounced_date + from email_actions + where action = 'bounce' + group by campaign_id, email_id +), + +opens as ( + -- look at the first open date + select campaign_id, email_id, min(action_date) as opened_date + from email_actions + where action = 'open' + group by campaign_id, email_id +), + +clicks as +( + -- look at the first click data + select campaign_id, email_id, min(action_date) as clicked_date + from email_actions + where action = 'click' + group by campaign_id, email_id +) + +select + s.email_id, sent_date, bounced_date, opened_date, clicked_date, + decode(bounced_date, null, 0, 1) as bounced, + decode(opened_date, null, 0, 1) as opened, + decode(clicked_date, null, 0, 1) as clicked +from sends s +left outer join bounces b + on s.email_id = b.email_id and + s.campaign_id = b.campaign_id +left outer join opens o + on s.email_id = o.email_id and + s.campaign_id = o.campaign_id +left outer join clicks c + on s.email_id = c.email_id and + s.campaign_id = c.campaign_id +order by 1, 2 + + diff --git a/models/mailchimp/mailchimp_lists.sql b/models/mailchimp/mailchimp_lists.sql new file mode 100644 index 0000000..84ae694 --- /dev/null +++ b/models/mailchimp/mailchimp_lists.sql @@ -0,0 +1,3 @@ +select distinct + id as list_id, name as list_name +from demo_data.mailchimp_lists \ No newline at end of file diff --git a/models/mailchimp/mailchimp_members.sql b/models/mailchimp/mailchimp_members.sql new file mode 100644 index 0000000..3fc2985 --- /dev/null +++ b/models/mailchimp/mailchimp_members.sql @@ -0,0 +1,3 @@ +select distinct + list_id, "timestamp_signup" as signup_date, id as email_id, email_address +from demo_data.mailchimp_members \ No newline at end of file diff --git a/models/mailchimp/mailchimp_sent_to.sql b/models/mailchimp/mailchimp_sent_to.sql new file mode 100644 index 0000000..e7437b5 --- /dev/null +++ b/models/mailchimp/mailchimp_sent_to.sql @@ -0,0 +1,7 @@ +select distinct + list_id, campaign_id, email_id, email_address, status, send_time as sent_date +from demo_data.mailchimp_sent_to a +-- add information about when the campaign was sent +inner join demo_data.mailchimp_campaigns b + on a.campaign_id = b.id + and a.list_id = b.recipients__list_id \ No newline at end of file From 6adcd8772da95ec5536d9d45b6d95f70aa1c88a7 Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Wed, 6 Apr 2016 14:33:34 -0400 Subject: [PATCH 71/88] removed mixpanel from the mailchimp branch --- analysis/mixpanel/mixpanel_event_cohorts.sql | 74 ------------------ analysis/mixpanel/mixpanel_funnel.sql | 75 ------------------- analysis/mixpanel/mixpanel_timeseries.sql | 12 --- models/mixpanel/mixpanel_engage.sql | 4 - models/mixpanel/mixpanel_export.sql | 5 -- .../mixpanel_export_with_sessions.sql | 17 ----- 6 files changed, 187 deletions(-) delete mode 100644 analysis/mixpanel/mixpanel_event_cohorts.sql delete mode 100644 analysis/mixpanel/mixpanel_funnel.sql delete mode 100644 analysis/mixpanel/mixpanel_timeseries.sql delete mode 100644 models/mixpanel/mixpanel_engage.sql delete mode 100644 models/mixpanel/mixpanel_export.sql delete mode 100644 models/mixpanel/mixpanel_export_with_sessions.sql diff --git a/analysis/mixpanel/mixpanel_event_cohorts.sql b/analysis/mixpanel/mixpanel_event_cohorts.sql deleted file mode 100644 index 48e2446..0000000 --- a/analysis/mixpanel/mixpanel_event_cohorts.sql +++ /dev/null @@ -1,74 +0,0 @@ -with -mixpanel_events as -( - select * - from {{env.schema}}.mixpanel_export -), - -first_event_data as -( - -- find all (user, date) records for the first event of interest - select - user_id, first_event_date - from - ( - -- pick the first occurence of the event of interest for cohorting purposes - select user_id, min(event_date) as first_event_date - from mixpanel_events - where - -- specifiy the first event of interest - event = 'event_1' - group by user_id - ) -), - -second_event_data as ( - -- find all (user, date) records for the second event of interest - select user_id, event_date as second_event_date - from mixpanel_events - where - -- specify the second event of interest - event = 'visualize impactful channels' -), - -combined_event_data as -( - -- get the table of first and second event dates for each user - select first_event_data.user_id, first_event_date, second_event_date - from first_event_data - inner join second_event_data - on first_event_data.user_id = second_event_data.user_id -), - -cohort_sizes as -( - -- compute cohort sizes - select - date_trunc('week', first_event_date)::date as cohort_date, - count(distinct user_id) as cohort_size - from combined_event_data - group by date_trunc('week', first_event_date) -), - -cohort_data as ( - select cohorts.cohort_date, cohort_size, user_id, days_to_second_event - from - ( - select - user_id, date_trunc('week', first_event_date)::date as cohort_date, - datediff(day, first_event_date, second_event_date) as days_to_second_event - from combined_event_data - ) cohorts - inner join cohort_sizes - on cohorts.cohort_date = cohort_sizes.cohort_date -) - -select - cohort_date, cohort_size, days_to_second_event, count(distinct user_id) as users, - count(distinct user_id)::float/cohort_size as portion_of_users -from cohort_data -group by cohort_date, cohort_size, days_to_second_event -order by cohort_date, cohort_size, days_to_second_event - - - diff --git a/analysis/mixpanel/mixpanel_funnel.sql b/analysis/mixpanel/mixpanel_funnel.sql deleted file mode 100644 index c689335..0000000 --- a/analysis/mixpanel/mixpanel_funnel.sql +++ /dev/null @@ -1,75 +0,0 @@ -with -source as ( - select * from {{env.schema}}.mixpanel_export -- change this view for your analysis -), - -step_1 as ( - select min(event_date) as event_date, user_id - from source - where event = 'event_1' -- filter by whichever columns you need - group by user_id -), - --- add more steps as you need. If you do add more steps, make sure to add a join below -step_2 as ( - select min(event_date) as event_date, user_id - from source - where event = 'event_2' - group by user_id -), - -step_3 as ( - select min(event_date) as event_date, user_id - from source - where event = 'event_3' - group by user_id -), - -step_4 as ( - select min(event_date) as event_date, user_id - from source - where event = 'event_4' - group by user_id -), - -step_5 as ( - select min(event_date) as event_date, user_id - from source - where event = 'event_5' - group by user_id -), - -temp_funnel as ( - - select 1 as funnel_idx, 'step_1' as funnel_step, count(distinct user_id) as num_current_step - from step_1 - union - select 2 as funnel_idx, 'step_2' as funnel_step, count(distinct user_id) as num_current_step - from step_2 - union - select 3 as funnel_idx, 'step_3' as funnel_step, count(distinct user_id) as num_current_step - from step_3 - union - select 4 as funnel_idx, 'step_4' as funnel_step, count(distinct user_id) as num_current_step - from step_4 - union - select 5 as funnel_idx, 'step_5' as funnel_step, count(distinct user_id) as num_current_step - from step_5 -) - - -select - funnel_step, num_current_step as funnel_count, - round(100.0*num_current_step/num_previous_step, 2) as pcnt_previous, - round(100.0*num_current_step/num_first_step, 2) as pcnt_overall -from -( - select - funnel_idx, funnel_step, num_current_step, - -- lag the funnel numbers to compute conversion ratios - lag(num_current_step) over(order by funnel_idx) as num_previous_step, - -- get the first step number to compute the overall conversion ratio - max(num_current_step) over() as num_first_step - from temp_funnel -) -order by funnel_idx diff --git a/analysis/mixpanel/mixpanel_timeseries.sql b/analysis/mixpanel/mixpanel_timeseries.sql deleted file mode 100644 index bb22a52..0000000 --- a/analysis/mixpanel/mixpanel_timeseries.sql +++ /dev/null @@ -1,12 +0,0 @@ -select - -- use second, minute, hour, day, week, month, quarter, etc - date_trunc('day', event_date) as period, count(*) period_count -from {{env.schema}}.mixpanel_export -/* ---select the time horizon and specific events here -where - --event_date > getdate() - interval '1 week' - --and ac_event = 'event_1' -*/ -group by period -order by period asc diff --git a/models/mixpanel/mixpanel_engage.sql b/models/mixpanel/mixpanel_engage.sql deleted file mode 100644 index 495f269..0000000 --- a/models/mixpanel/mixpanel_engage.sql +++ /dev/null @@ -1,4 +0,0 @@ -select - mp_reserved_created as user_created, - mp_reserved_distinct_id as user_id -from demo_data.mixpanel_engage \ No newline at end of file diff --git a/models/mixpanel/mixpanel_export.sql b/models/mixpanel/mixpanel_export.sql deleted file mode 100644 index b8430c1..0000000 --- a/models/mixpanel/mixpanel_export.sql +++ /dev/null @@ -1,5 +0,0 @@ -select - -- convert unix time to timestamp - (timestamp 'epoch' + time * interval '1 Second') as event_date, - event, distinct_id as user_id -from demo_data.mixpanel_export \ No newline at end of file diff --git a/models/mixpanel/mixpanel_export_with_sessions.sql b/models/mixpanel/mixpanel_export_with_sessions.sql deleted file mode 100644 index 52ca469..0000000 --- a/models/mixpanel/mixpanel_export_with_sessions.sql +++ /dev/null @@ -1,17 +0,0 @@ --- a new session is defined after 30 minutes of inactivity -with new_sessions as -( - select - case - when extract(epoch from event_date) - lag(extract(epoch from event_date)) - over (partition by user_id order by event_date) >= 30 * 60 - then 1 - else 0 - end as new_session, * - from {{env.schema}}.mixpanel_export -) - --- make sure the first sessions is marked 1 and not 0 -select sum(new_session) - over (partition by user_id order by event_date rows unbounded preceding) + 1 as session_idx, * -from new_sessions \ No newline at end of file From 3e623c751e013a4f549212e83916d55ad00dfd59 Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Fri, 8 Apr 2016 13:51:02 -0400 Subject: [PATCH 72/88] added base models for every action and added gains and losses --- .../mailchimp/mailchimp_email_timeseries.sql | 24 +++++--- analysis/mailchimp/mailchimp_gains_losses.sql | 47 +++++++++++++++ models/mailchimp/mailchimp_all_events.sql | 52 +++++++++++++++++ models/mailchimp/mailchimp_bounces.sql | 6 ++ models/mailchimp/mailchimp_campaigns.sql | 2 +- models/mailchimp/mailchimp_clicks.sql | 3 + models/mailchimp/mailchimp_email_summary.sql | 58 +++++++++++++++++++ .../mailchimp_emails_denormalized.sql | 53 ----------------- models/mailchimp/mailchimp_lists.sql | 2 +- models/mailchimp/mailchimp_members.sql | 2 +- models/mailchimp/mailchimp_opens.sql | 3 + models/mailchimp/mailchimp_sends.sql | 6 ++ models/mailchimp/mailchimp_sent_to.sql | 10 +--- models/mailchimp/mailchimp_unsubscribes.sql | 2 + 14 files changed, 198 insertions(+), 72 deletions(-) create mode 100644 analysis/mailchimp/mailchimp_gains_losses.sql create mode 100644 models/mailchimp/mailchimp_all_events.sql create mode 100644 models/mailchimp/mailchimp_bounces.sql create mode 100644 models/mailchimp/mailchimp_clicks.sql create mode 100644 models/mailchimp/mailchimp_email_summary.sql delete mode 100644 models/mailchimp/mailchimp_emails_denormalized.sql create mode 100644 models/mailchimp/mailchimp_opens.sql create mode 100644 models/mailchimp/mailchimp_sends.sql create mode 100644 models/mailchimp/mailchimp_unsubscribes.sql diff --git a/analysis/mailchimp/mailchimp_email_timeseries.sql b/analysis/mailchimp/mailchimp_email_timeseries.sql index aa877f2..c6b2a1e 100644 --- a/analysis/mailchimp/mailchimp_email_timeseries.sql +++ b/analysis/mailchimp/mailchimp_email_timeseries.sql @@ -1,14 +1,20 @@ with events as ( - select * from {{env.schema}}.mailchimp_emails_denormalized + select + campaign_id, email_id, sent_date, + decode(hard_bounced_date, null, 0, 1) as hard_bounced, + decode(first_opened_date, null, 0, 1) as opened, + decode(first_clicked_date, null, 0, 1) as clicked, + decode(unsubscribed_date, null, 0, 1) as unsubscribed + from {{env.schema}}.mailchimp_email_summary ) select - date_trunc('month', sent_date) as mnth, - count(*) as sends, - sum(opened) as opens, - sum(clicked) as clicks, - avg(opened::float) as open_rate, - avg(clicked::float) as click_rate + date_trunc('month', sent_date) as month, + count(*) as sends, + sum(opened) as opens, + sum(clicked) as clicks, + avg(opened::float) as open_rate, + avg(clicked::float) as click_rate from events -group by 1 -order by 1 \ No newline at end of file +group by month +order by month \ No newline at end of file diff --git a/analysis/mailchimp/mailchimp_gains_losses.sql b/analysis/mailchimp/mailchimp_gains_losses.sql new file mode 100644 index 0000000..2df4e7a --- /dev/null +++ b/analysis/mailchimp/mailchimp_gains_losses.sql @@ -0,0 +1,47 @@ +with +email_summary as +( + select * + from {{env.schema}}.mailchimp_email_summary +), + +gains as ( + select date_trunc('month', signup_date) as month, count(*) total_gains + from {{env.schema}}.mailchimp_members + group by date_trunc('month', signup_date) +), + +losses as ( + select month, count(*) as total_losses + from + ( + -- find everyone who unsubscribed + select date_trunc('month', unsubscribed_date) as month + from email_summary + where unsubscribed_date is not null + -- add everyone with a hard bounce + union + select date_trunc('month', hard_bounced_date) as month + from email_summary + where hard_bounced_date is not null + ) + group by month +), + +months as ( + select month + from gains + union + select month + from losses +) + +select + months.month, nvl(total_gains,0) as total_gains, + nvl(total_losses,0) as total_losses +from months +left outer join gains + on months.month = gains.month +left outer join losses + on months.month = losses.month +order by months.month \ No newline at end of file diff --git a/models/mailchimp/mailchimp_all_events.sql b/models/mailchimp/mailchimp_all_events.sql new file mode 100644 index 0000000..566ae1f --- /dev/null +++ b/models/mailchimp/mailchimp_all_events.sql @@ -0,0 +1,52 @@ +with +sends as ( + select campaign_id, email_id, 'sent' as event_action, sent_date as event_date + from {{env.schema}}.mailchimp_sends +), + +soft_bounces as ( + select campaign_id, email_id, 'soft bounce' as event_action, bounced_date as event_date + from {{env.schema}}.mailchimp_bounces + where bounce_type = 'soft' +), + +hard_bounces as ( + select campaign_id, email_id, 'hard bounce' as event_action, bounced_date as event_date + from {{env.schema}}.mailchimp_bounces + where bounce_type = 'hard' +), + +opens as ( + select campaign_id, email_id, 'opened' as event_action, opened_date as event_date + from {{env.schema}}.mailchimp_opens +), + +clicks as ( + select campaign_id, email_id, 'clicked' as event_action, clicked_date as event_date + from {{env.schema}}.mailchimp_clicks +), + +unsubscribes as ( + select campaign_id, email_id, 'unsubscribed' as event_action, unsubscribed_date as event_date + from {{env.schema}}.mailchimp_unsubscribes +) + +select * +from sends +union +select * +from soft_bounces +union +select * +from hard_bounces +union +select * +from opens +union +select * +from clicks +union +select * +from unsubscribes + + diff --git a/models/mailchimp/mailchimp_bounces.sql b/models/mailchimp/mailchimp_bounces.sql new file mode 100644 index 0000000..c24e6e2 --- /dev/null +++ b/models/mailchimp/mailchimp_bounces.sql @@ -0,0 +1,6 @@ +select a.campaign_id, a.email_id, action_date as bounced_date, status as bounce_type +from {{env.schema}}.mailchimp_email_actions a +inner join {{env.schema}}.mailchimp_sent_to b + on a.campaign_id = b.campaign_id + and a.email_id = b.email_id +where action = 'bounce' \ No newline at end of file diff --git a/models/mailchimp/mailchimp_campaigns.sql b/models/mailchimp/mailchimp_campaigns.sql index c246487..76ed116 100644 --- a/models/mailchimp/mailchimp_campaigns.sql +++ b/models/mailchimp/mailchimp_campaigns.sql @@ -1,3 +1,3 @@ -select distinct +select recipients__list_id as list_id, id as campaign_id, send_time as sent_date from demo_data.mailchimp_campaigns \ No newline at end of file diff --git a/models/mailchimp/mailchimp_clicks.sql b/models/mailchimp/mailchimp_clicks.sql new file mode 100644 index 0000000..dca335f --- /dev/null +++ b/models/mailchimp/mailchimp_clicks.sql @@ -0,0 +1,3 @@ +select campaign_id, email_id, action_date as clicked_date +from {{env.schema}}.mailchimp_email_actions +where action = 'click' \ No newline at end of file diff --git a/models/mailchimp/mailchimp_email_summary.sql b/models/mailchimp/mailchimp_email_summary.sql new file mode 100644 index 0000000..8da2700 --- /dev/null +++ b/models/mailchimp/mailchimp_email_summary.sql @@ -0,0 +1,58 @@ +with +sends as ( + select campaign_id, email_id, sent_date + from {{env.schema}}.mailchimp_sends +), + +hard_bounces as ( + select campaign_id, email_id, min(bounced_date) as hard_bounced_date + from {{env.schema}}.mailchimp_bounces + where bounce_type = 'hard' + group by campaign_id, email_id +), + +opens as ( + select + campaign_id, email_id, min(opened_date) as first_opened_date, + max(opened_date) as last_opened_date, count(*) as total_opens + from {{env.schema}}.mailchimp_opens + group by campaign_id, email_id +), + +clicks as ( + select + campaign_id, email_id, min(clicked_date) as first_clicked_date, + max(clicked_date) as last_clicked_date, count(*) as total_clicks + from {{env.schema}}.mailchimp_clicks + group by campaign_id, email_id +), + +unsubscribes as ( + select campaign_id, email_id, unsubscribed_date + from {{env.schema}}.mailchimp_unsubscribes +) + +select + s.campaign_id, s.email_id, sent_date, hard_bounced_date, first_opened_date, + last_opened_date, total_opens, first_clicked_date, last_clicked_date, + total_clicks, unsubscribed_date--, + --decode(bounced_date, null, 0, 1) as hard_bounced, + --decode(opened_date, null, 0, 1) as opened, + --decode(clicked_date, null, 0, 1) as clicked, + --decode(unsubscribed_date, null, 0, 1) as unsubscribed +from sends s +left outer join hard_bounces b + on s.email_id = b.email_id and + s.campaign_id = b.campaign_id +left outer join opens o + on s.email_id = o.email_id and + s.campaign_id = o.campaign_id +left outer join clicks c + on s.email_id = c.email_id and + s.campaign_id = c.campaign_id +left outer join unsubscribes u + on s.email_id = u.email_id and + s.campaign_id = u.campaign_id +order by email_id, sent_date + + diff --git a/models/mailchimp/mailchimp_emails_denormalized.sql b/models/mailchimp/mailchimp_emails_denormalized.sql deleted file mode 100644 index 525bf07..0000000 --- a/models/mailchimp/mailchimp_emails_denormalized.sql +++ /dev/null @@ -1,53 +0,0 @@ -with email_actions as ( - select * - from {{env.schema}}.mailchimp_email_actions -), - -sends as ( - select campaign_id, email_id, sent_date - from {{env.schema}}.mailchimp_sent_to -), - -bounces as -( - select campaign_id, email_id, min(action_date) as bounced_date - from email_actions - where action = 'bounce' - group by campaign_id, email_id -), - -opens as ( - -- look at the first open date - select campaign_id, email_id, min(action_date) as opened_date - from email_actions - where action = 'open' - group by campaign_id, email_id -), - -clicks as -( - -- look at the first click data - select campaign_id, email_id, min(action_date) as clicked_date - from email_actions - where action = 'click' - group by campaign_id, email_id -) - -select - s.email_id, sent_date, bounced_date, opened_date, clicked_date, - decode(bounced_date, null, 0, 1) as bounced, - decode(opened_date, null, 0, 1) as opened, - decode(clicked_date, null, 0, 1) as clicked -from sends s -left outer join bounces b - on s.email_id = b.email_id and - s.campaign_id = b.campaign_id -left outer join opens o - on s.email_id = o.email_id and - s.campaign_id = o.campaign_id -left outer join clicks c - on s.email_id = c.email_id and - s.campaign_id = c.campaign_id -order by 1, 2 - - diff --git a/models/mailchimp/mailchimp_lists.sql b/models/mailchimp/mailchimp_lists.sql index 84ae694..6762eed 100644 --- a/models/mailchimp/mailchimp_lists.sql +++ b/models/mailchimp/mailchimp_lists.sql @@ -1,3 +1,3 @@ -select distinct +select id as list_id, name as list_name from demo_data.mailchimp_lists \ No newline at end of file diff --git a/models/mailchimp/mailchimp_members.sql b/models/mailchimp/mailchimp_members.sql index 3fc2985..927513e 100644 --- a/models/mailchimp/mailchimp_members.sql +++ b/models/mailchimp/mailchimp_members.sql @@ -1,3 +1,3 @@ -select distinct +select list_id, "timestamp_signup" as signup_date, id as email_id, email_address from demo_data.mailchimp_members \ No newline at end of file diff --git a/models/mailchimp/mailchimp_opens.sql b/models/mailchimp/mailchimp_opens.sql new file mode 100644 index 0000000..59865a6 --- /dev/null +++ b/models/mailchimp/mailchimp_opens.sql @@ -0,0 +1,3 @@ +select campaign_id, email_id, action_date as opened_date +from {{env.schema}}.mailchimp_email_actions +where action = 'open' \ No newline at end of file diff --git a/models/mailchimp/mailchimp_sends.sql b/models/mailchimp/mailchimp_sends.sql new file mode 100644 index 0000000..ca553c2 --- /dev/null +++ b/models/mailchimp/mailchimp_sends.sql @@ -0,0 +1,6 @@ +select a.campaign_id, a.email_id, sent_date +from {{env.schema}}.mailchimp_sent_to a +-- add information about when the campaign was sent +inner join {{env.schema}}.mailchimp_campaigns b + on a.campaign_id = b.campaign_id + and a.list_id = b.list_id \ No newline at end of file diff --git a/models/mailchimp/mailchimp_sent_to.sql b/models/mailchimp/mailchimp_sent_to.sql index e7437b5..6207c37 100644 --- a/models/mailchimp/mailchimp_sent_to.sql +++ b/models/mailchimp/mailchimp_sent_to.sql @@ -1,7 +1,3 @@ -select distinct - list_id, campaign_id, email_id, email_address, status, send_time as sent_date -from demo_data.mailchimp_sent_to a --- add information about when the campaign was sent -inner join demo_data.mailchimp_campaigns b - on a.campaign_id = b.id - and a.list_id = b.recipients__list_id \ No newline at end of file +select + list_id, campaign_id, email_id, email_address, status +from demo_data.mailchimp_sent_to \ No newline at end of file diff --git a/models/mailchimp/mailchimp_unsubscribes.sql b/models/mailchimp/mailchimp_unsubscribes.sql new file mode 100644 index 0000000..3de8453 --- /dev/null +++ b/models/mailchimp/mailchimp_unsubscribes.sql @@ -0,0 +1,2 @@ +select campaign_id, email_id, "timestamp" as unsubscribed_date +from demo_data.mailchimp_unsubscribes \ No newline at end of file From 97db355b4142a06745b78e0178b9e51880ccd102 Mon Sep 17 00:00:00 2001 From: Yevgeniy Date: Mon, 11 Apr 2016 11:59:57 -0400 Subject: [PATCH 73/88] updated gains_losses --- analysis/mailchimp/mailchimp_gains_losses.sql | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/analysis/mailchimp/mailchimp_gains_losses.sql b/analysis/mailchimp/mailchimp_gains_losses.sql index 2df4e7a..b0a4237 100644 --- a/analysis/mailchimp/mailchimp_gains_losses.sql +++ b/analysis/mailchimp/mailchimp_gains_losses.sql @@ -1,29 +1,43 @@ with -email_summary as -( - select * - from {{env.schema}}.mailchimp_email_summary -), - gains as ( select date_trunc('month', signup_date) as month, count(*) total_gains from {{env.schema}}.mailchimp_members group by date_trunc('month', signup_date) ), +hard_bounces as +( + -- get the first hard bounce date for each email + select email_id, min(bounced_date) as event_date + from {{env.schema}}.mailchimp_bounces + where bounce_type = 'hard' + group by email_id +), + +unsubscribes as +( + -- get the first unsubscribed date for each email + select email_id, min(unsubscribed_date) as event_date + from {{env.schema}}.mailchimp_unsubscribes + group by email_id +), + losses as ( - select month, count(*) as total_losses + -- count losses for each month + select date_trunc('month', event_date) as month, count(*) as total_losses from ( - -- find everyone who unsubscribed - select date_trunc('month', unsubscribed_date) as month - from email_summary - where unsubscribed_date is not null - -- add everyone with a hard bounce - union - select date_trunc('month', hard_bounced_date) as month - from email_summary - where hard_bounced_date is not null + -- select the first of unsubscribe or hard bounce + select email_id, min(event_date) as event_date + from + ( + select email_id, event_date + from hard_bounces + union + select email_id, event_date + from unsubscribes + ) + group by email_id ) group by month ), From ec70d0197326a9c6cc1dab600130713d31be3038 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Thu, 14 Apr 2016 14:21:00 -0400 Subject: [PATCH 74/88] updates to mailchimp analysis for blog post --- analysis/mailchimp/mailchimp_best_times.sql | 23 +++++++++++++++ .../mailchimp/mailchimp_email_timeseries.sql | 9 ++---- analysis/mailchimp/mailchimp_gains_losses.sql | 29 +++++++++++++------ analysis/mailchimp/mailchimp_most_engaged.sql | 13 +++++++++ models/mailchimp/mailchimp_email_summary.sql | 14 ++++----- 5 files changed, 64 insertions(+), 24 deletions(-) create mode 100644 analysis/mailchimp/mailchimp_best_times.sql create mode 100644 analysis/mailchimp/mailchimp_most_engaged.sql diff --git a/analysis/mailchimp/mailchimp_best_times.sql b/analysis/mailchimp/mailchimp_best_times.sql new file mode 100644 index 0000000..579a1e6 --- /dev/null +++ b/analysis/mailchimp/mailchimp_best_times.sql @@ -0,0 +1,23 @@ +with email_summary as ( + + select * + from {{env.schema}}.mailchimp_email_summary + +), best_day_of_week as ( + + select date_part(dow, sent_date), avg(opened::float) + from email_summary + group by 1 + order by 1 + +), best_hour_of_day as ( + + select date_part(hr, sent_date), avg(opened::float) + from email_summary + group by 1 + order by 1 + +) + +select * +from best_day_of_week diff --git a/analysis/mailchimp/mailchimp_email_timeseries.sql b/analysis/mailchimp/mailchimp_email_timeseries.sql index c6b2a1e..0230e86 100644 --- a/analysis/mailchimp/mailchimp_email_timeseries.sql +++ b/analysis/mailchimp/mailchimp_email_timeseries.sql @@ -1,10 +1,5 @@ with events as ( - select - campaign_id, email_id, sent_date, - decode(hard_bounced_date, null, 0, 1) as hard_bounced, - decode(first_opened_date, null, 0, 1) as opened, - decode(first_clicked_date, null, 0, 1) as clicked, - decode(unsubscribed_date, null, 0, 1) as unsubscribed + select * from {{env.schema}}.mailchimp_email_summary ) @@ -17,4 +12,4 @@ select avg(clicked::float) as click_rate from events group by month -order by month \ No newline at end of file +order by month diff --git a/analysis/mailchimp/mailchimp_gains_losses.sql b/analysis/mailchimp/mailchimp_gains_losses.sql index b0a4237..f4c3114 100644 --- a/analysis/mailchimp/mailchimp_gains_losses.sql +++ b/analysis/mailchimp/mailchimp_gains_losses.sql @@ -48,14 +48,25 @@ months as ( union select month from losses +), + +monthly_gains_and_losses as ( + + select + months.month, nvl(total_gains,0) as month_gains, + nvl(total_losses,0) as month_losses + from months + left outer join gains + on months.month = gains.month + left outer join losses + on months.month = losses.month + order by months.month + ) -select - months.month, nvl(total_gains,0) as total_gains, - nvl(total_losses,0) as total_losses -from months -left outer join gains - on months.month = gains.month -left outer join losses - on months.month = losses.month -order by months.month \ No newline at end of file +select month, month_gains, month_losses * -1 as month_losses, month_gains - month_losses as net_member_growth, + sum(month_gains) over (order by month rows unbounded preceding) as cum_gains, + sum(month_losses) over (order by month rows unbounded preceding) as cum_losses, + sum(month_gains) over (order by month rows unbounded preceding) - + sum(month_losses) over (order by month rows unbounded preceding) as active_members +from monthly_gains_and_losses diff --git a/analysis/mailchimp/mailchimp_most_engaged.sql b/analysis/mailchimp/mailchimp_most_engaged.sql new file mode 100644 index 0000000..439add1 --- /dev/null +++ b/analysis/mailchimp/mailchimp_most_engaged.sql @@ -0,0 +1,13 @@ +with email_summary as ( + + select * + from {{env.schema}}.mailchimp_email_summary + +) + +select email_id, sum(clicked)::float / sum(opened) + from email_summary + group by 1 +having sum(opened) > 5 + order by 2 desc + limit 100 diff --git a/models/mailchimp/mailchimp_email_summary.sql b/models/mailchimp/mailchimp_email_summary.sql index 8da2700..00f2129 100644 --- a/models/mailchimp/mailchimp_email_summary.sql +++ b/models/mailchimp/mailchimp_email_summary.sql @@ -1,4 +1,4 @@ -with +with sends as ( select campaign_id, email_id, sent_date from {{env.schema}}.mailchimp_sends @@ -35,11 +35,11 @@ unsubscribes as ( select s.campaign_id, s.email_id, sent_date, hard_bounced_date, first_opened_date, last_opened_date, total_opens, first_clicked_date, last_clicked_date, - total_clicks, unsubscribed_date--, - --decode(bounced_date, null, 0, 1) as hard_bounced, - --decode(opened_date, null, 0, 1) as opened, - --decode(clicked_date, null, 0, 1) as clicked, - --decode(unsubscribed_date, null, 0, 1) as unsubscribed + total_clicks, unsubscribed_date, + decode(hard_bounced_date, null, 0, 1) as hard_bounced, + decode(first_opened_date, null, 0, 1) as opened, + decode(first_clicked_date, null, 0, 1) as clicked, + decode(unsubscribed_date, null, 0, 1) as unsubscribed from sends s left outer join hard_bounces b on s.email_id = b.email_id and @@ -54,5 +54,3 @@ left outer join unsubscribes u on s.email_id = u.email_id and s.campaign_id = u.campaign_id order by email_id, sent_date - - From 0ff459c95c629e37e9a9c060c34a2fced15dcbc4 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Fri, 15 Apr 2016 12:58:11 -0400 Subject: [PATCH 75/88] updates based on pr --- analysis/mailchimp/mailchimp_gains_losses.sql | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/analysis/mailchimp/mailchimp_gains_losses.sql b/analysis/mailchimp/mailchimp_gains_losses.sql index f4c3114..21e7d95 100644 --- a/analysis/mailchimp/mailchimp_gains_losses.sql +++ b/analysis/mailchimp/mailchimp_gains_losses.sql @@ -54,7 +54,7 @@ monthly_gains_and_losses as ( select months.month, nvl(total_gains,0) as month_gains, - nvl(total_losses,0) as month_losses + -nvl(total_losses,0) as month_losses from months left outer join gains on months.month = gains.month @@ -64,9 +64,8 @@ monthly_gains_and_losses as ( ) -select month, month_gains, month_losses * -1 as month_losses, month_gains - month_losses as net_member_growth, +select month, month_gains, month_losses, month_gains + month_losses as net_member_growth, sum(month_gains) over (order by month rows unbounded preceding) as cum_gains, sum(month_losses) over (order by month rows unbounded preceding) as cum_losses, - sum(month_gains) over (order by month rows unbounded preceding) - - sum(month_losses) over (order by month rows unbounded preceding) as active_members + sum(month_gains + month_losses) over (order by month rows unbounded preceding) as active_members from monthly_gains_and_losses From 6e96fa5fae494a393f49dfa330a79979b13e7d97 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Fri, 15 Apr 2016 13:05:50 -0400 Subject: [PATCH 76/88] spacing! --- analysis/mailchimp/mailchimp_most_engaged.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/analysis/mailchimp/mailchimp_most_engaged.sql b/analysis/mailchimp/mailchimp_most_engaged.sql index 439add1..05a7534 100644 --- a/analysis/mailchimp/mailchimp_most_engaged.sql +++ b/analysis/mailchimp/mailchimp_most_engaged.sql @@ -6,8 +6,8 @@ with email_summary as ( ) select email_id, sum(clicked)::float / sum(opened) - from email_summary - group by 1 +from email_summary +group by 1 having sum(opened) > 5 - order by 2 desc - limit 100 +order by 2 desc +limit 100 From 3af6b343fa7ce9e2715ac4b1881cf1e14c1ce49a Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 3 May 2016 14:29:21 -0400 Subject: [PATCH 77/88] don't explicity reference env.schema in model defs --- models/email/emails_denormalized.sql | 2 +- models/mailchimp/mailchimp_all_events.sql | 12 ++++++------ models/mailchimp/mailchimp_bounces.sql | 4 ++-- models/mailchimp/mailchimp_clicks.sql | 2 +- models/mailchimp/mailchimp_email_summary.sql | 10 +++++----- models/mailchimp/mailchimp_opens.sql | 2 +- models/mailchimp/mailchimp_sends.sql | 4 ++-- models/pardot/pardot_emails.sql | 2 +- models/pardot/pardot_tests.sql | 2 +- models/pardot/pardot_visitoractivity.sql | 4 ++-- models/stripe/stripe_invoices_cleaned.sql | 2 +- models/stripe/stripe_invoices_transformed.sql | 6 +++--- models/trello/trello_model_tests.sql | 4 ++-- .../zuora_subscriptions_w_charges_and_amendments.sql | 10 +++++----- 14 files changed, 33 insertions(+), 33 deletions(-) diff --git a/models/email/emails_denormalized.sql b/models/email/emails_denormalized.sql index a483de7..81e6192 100644 --- a/models/email/emails_denormalized.sql +++ b/models/email/emails_denormalized.sql @@ -1,6 +1,6 @@ with events as ( - select * from {{env.schema}}.pardot_emails + select * from {{load('pardot_emails')}} ), diff --git a/models/mailchimp/mailchimp_all_events.sql b/models/mailchimp/mailchimp_all_events.sql index 566ae1f..d62074c 100644 --- a/models/mailchimp/mailchimp_all_events.sql +++ b/models/mailchimp/mailchimp_all_events.sql @@ -1,34 +1,34 @@ with sends as ( select campaign_id, email_id, 'sent' as event_action, sent_date as event_date - from {{env.schema}}.mailchimp_sends + from {{load('mailchimp_sends')}} ), soft_bounces as ( select campaign_id, email_id, 'soft bounce' as event_action, bounced_date as event_date - from {{env.schema}}.mailchimp_bounces + from {{load('mailchimp_bounces')}} where bounce_type = 'soft' ), hard_bounces as ( select campaign_id, email_id, 'hard bounce' as event_action, bounced_date as event_date - from {{env.schema}}.mailchimp_bounces + from {{load('mailchimp_bounces')}} where bounce_type = 'hard' ), opens as ( select campaign_id, email_id, 'opened' as event_action, opened_date as event_date - from {{env.schema}}.mailchimp_opens + from {{load('mailchimp_opens')}} ), clicks as ( select campaign_id, email_id, 'clicked' as event_action, clicked_date as event_date - from {{env.schema}}.mailchimp_clicks + from {{load('mailchimp_clicks')}} ), unsubscribes as ( select campaign_id, email_id, 'unsubscribed' as event_action, unsubscribed_date as event_date - from {{env.schema}}.mailchimp_unsubscribes + from {{load('mailchimp_unsubscribes')}} ) select * diff --git a/models/mailchimp/mailchimp_bounces.sql b/models/mailchimp/mailchimp_bounces.sql index c24e6e2..cad4950 100644 --- a/models/mailchimp/mailchimp_bounces.sql +++ b/models/mailchimp/mailchimp_bounces.sql @@ -1,6 +1,6 @@ select a.campaign_id, a.email_id, action_date as bounced_date, status as bounce_type -from {{env.schema}}.mailchimp_email_actions a -inner join {{env.schema}}.mailchimp_sent_to b +from {{load('mailchimp_email_actions')}} a +inner join {{load('mailchimp_sent_to')}} b on a.campaign_id = b.campaign_id and a.email_id = b.email_id where action = 'bounce' \ No newline at end of file diff --git a/models/mailchimp/mailchimp_clicks.sql b/models/mailchimp/mailchimp_clicks.sql index dca335f..3c78a4a 100644 --- a/models/mailchimp/mailchimp_clicks.sql +++ b/models/mailchimp/mailchimp_clicks.sql @@ -1,3 +1,3 @@ select campaign_id, email_id, action_date as clicked_date -from {{env.schema}}.mailchimp_email_actions +from {{load('mailchimp_email_actions')}} where action = 'click' \ No newline at end of file diff --git a/models/mailchimp/mailchimp_email_summary.sql b/models/mailchimp/mailchimp_email_summary.sql index 00f2129..0da1d4b 100644 --- a/models/mailchimp/mailchimp_email_summary.sql +++ b/models/mailchimp/mailchimp_email_summary.sql @@ -1,12 +1,12 @@ with sends as ( select campaign_id, email_id, sent_date - from {{env.schema}}.mailchimp_sends + from {{load('mailchimp_sends')}} ), hard_bounces as ( select campaign_id, email_id, min(bounced_date) as hard_bounced_date - from {{env.schema}}.mailchimp_bounces + from {{load('mailchimp_bounces')}} where bounce_type = 'hard' group by campaign_id, email_id ), @@ -15,7 +15,7 @@ opens as ( select campaign_id, email_id, min(opened_date) as first_opened_date, max(opened_date) as last_opened_date, count(*) as total_opens - from {{env.schema}}.mailchimp_opens + from {{load('mailchimp_opens')}} group by campaign_id, email_id ), @@ -23,13 +23,13 @@ clicks as ( select campaign_id, email_id, min(clicked_date) as first_clicked_date, max(clicked_date) as last_clicked_date, count(*) as total_clicks - from {{env.schema}}.mailchimp_clicks + from {{load('mailchimp_clicks')}} group by campaign_id, email_id ), unsubscribes as ( select campaign_id, email_id, unsubscribed_date - from {{env.schema}}.mailchimp_unsubscribes + from {{load('mailchimp_unsubscribes')}} ) select diff --git a/models/mailchimp/mailchimp_opens.sql b/models/mailchimp/mailchimp_opens.sql index 59865a6..1b8b4df 100644 --- a/models/mailchimp/mailchimp_opens.sql +++ b/models/mailchimp/mailchimp_opens.sql @@ -1,3 +1,3 @@ select campaign_id, email_id, action_date as opened_date -from {{env.schema}}.mailchimp_email_actions +from {{load('mailchimp_email_actions')}} where action = 'open' \ No newline at end of file diff --git a/models/mailchimp/mailchimp_sends.sql b/models/mailchimp/mailchimp_sends.sql index ca553c2..2261bf9 100644 --- a/models/mailchimp/mailchimp_sends.sql +++ b/models/mailchimp/mailchimp_sends.sql @@ -1,6 +1,6 @@ select a.campaign_id, a.email_id, sent_date -from {{env.schema}}.mailchimp_sent_to a +from {{load('mailchimp_sent_to')}} a -- add information about when the campaign was sent -inner join {{env.schema}}.mailchimp_campaigns b +inner join {{load('mailchimp_campaigns')}} b on a.campaign_id = b.campaign_id and a.list_id = b.list_id \ No newline at end of file diff --git a/models/pardot/pardot_emails.sql b/models/pardot/pardot_emails.sql index b8308f1..60df61c 100644 --- a/models/pardot/pardot_emails.sql +++ b/models/pardot/pardot_emails.sql @@ -5,5 +5,5 @@ data necessary to conform to the extended interface. */ select "@timestamp", "@event", "@user_id", email_id as "@email_id", details as "@subject" - from {{env.schema}}.pardot_visitoractivity + from {{load('pardot_visitoractivity')}} where "@event" in ('email sent', 'email opened', 'email click') diff --git a/models/pardot/pardot_tests.sql b/models/pardot/pardot_tests.sql index 20155df..76d60ef 100644 --- a/models/pardot/pardot_tests.sql +++ b/models/pardot/pardot_tests.sql @@ -2,7 +2,7 @@ select 'visitoractivity_fresher_than_one_day' as name, 'Most recent visitoractivity entry is no more than one day old' as description, max("@timestamp"::timestamp) > current_date - '1 day'::interval as result -from {{env.schema}}.pardot_visitoractivity +from {{load('pardot_visitoractivity')}} diff --git a/models/pardot/pardot_visitoractivity.sql b/models/pardot/pardot_visitoractivity.sql index f56509a..940bf84 100644 --- a/models/pardot/pardot_visitoractivity.sql +++ b/models/pardot/pardot_visitoractivity.sql @@ -9,7 +9,7 @@ select va.* from olga_pardot.visitoractivity va - inner join {{env.schema}}.pardot_visitoractivity_events_meta e + inner join {{load('pardot_visitoractivity_events_meta')}} e on va."type" = e."type" and va.type_name = e.type_name - inner join {{env.schema}}.pardot_visitoractivity_types_meta t + inner join {{load('pardot_visitoractivity_types_meta')}} t on va."type" = t."type" diff --git a/models/stripe/stripe_invoices_cleaned.sql b/models/stripe/stripe_invoices_cleaned.sql index fa38136..0d07205 100644 --- a/models/stripe/stripe_invoices_cleaned.sql +++ b/models/stripe/stripe_invoices_cleaned.sql @@ -9,6 +9,6 @@ select timestamp 'epoch' + period_start * interval '1 Second' as period_start, timestamp 'epoch' + period_end * interval '1 Second' as period_end from - {{env.schema}}.stripe_invoices + {{load('stripe_invoices')}} diff --git a/models/stripe/stripe_invoices_transformed.sql b/models/stripe/stripe_invoices_transformed.sql index 2c824a1..557be7e 100644 --- a/models/stripe/stripe_invoices_transformed.sql +++ b/models/stripe/stripe_invoices_transformed.sql @@ -2,7 +2,7 @@ with invoices as ( select * - from {{env.schema}}.stripe_invoices_cleaned + from {{load('stripe_invoices_cleaned')}} where paid is true and forgiven is false @@ -55,7 +55,7 @@ from customer_dates d on d.date_month >= date_trunc('month', i.period_start) and d.date_month < date_trunc('month', i.period_end) and d.customer = i.customer - left outer join {{env.schema}}.stripe_subscriptions s on i.subscription_id = s.id - left outer join {{env.schema}}.stripe_plans p on s.plan_id = p.id + left outer join {{load('stripe_subscriptions')}} s on i.subscription_id = s.id + left outer join {{load('stripe_plans')}} p on s.plan_id = p.id diff --git a/models/trello/trello_model_tests.sql b/models/trello/trello_model_tests.sql index 9f9840b..ad9dec6 100644 --- a/models/trello/trello_model_tests.sql +++ b/models/trello/trello_model_tests.sql @@ -1,7 +1,7 @@ with null_boards_or_lists as ( select id - from {{env.schema}}.trello_card_location + from {{load('trello_card_location')}} where data__board__id is null or data__list__id is null @@ -18,4 +18,4 @@ select 'fresher_than_one_day', 'Most recent entry is no more than one day old', max(date::timestamp) > current_date - '1 day'::interval -from {{env.schema}}.trello_card_location +from {{load('trello_card_location')}} diff --git a/models/zuora/zuora_subscriptions_w_charges_and_amendments.sql b/models/zuora/zuora_subscriptions_w_charges_and_amendments.sql index 4da0efa..bc8453a 100644 --- a/models/zuora/zuora_subscriptions_w_charges_and_amendments.sql +++ b/models/zuora/zuora_subscriptions_w_charges_and_amendments.sql @@ -5,11 +5,11 @@ with subscr_w_amendments as account_number, acc.account_id, sub.subscr_id, subscr_name, subscr_status, subscr_term_type, subscr_start, subscr_end, subscr_version, amend_id, amend_start - from {{env.schema}}.zuora_account acc - inner join {{env.schema}}.zuora_subscription sub + from {{load('zuora_account')}} acc + inner join {{load('zuora_subscription')}} sub on acc.account_id = sub.account_id -- add ammendments - left outer join {{env.schema}}.zuora_amendment amend + left outer join {{load('zuora_amendment')}} amend on sub.subscr_id = amend.subscr_id ) @@ -21,7 +21,7 @@ select min(subscr_start) over() as first_subscr, "@mrr" as mrr from subscr_w_amendments sub -inner join {{env.schema}}.zuora_rate_plan rp +inner join {{load('zuora_rate_plan')}} rp on rp.subscr_id = sub.subscr_id -inner join {{env.schema}}.zuora_rate_plan_charge rpc +inner join {{load('zuora_rate_plan_charge')}} rpc on rpc.rate_plan_id = rp.rate_plan_id From 0f2ca78f89791717862b770b20d27059decf48bf Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 3 May 2016 16:15:43 -0400 Subject: [PATCH 78/88] replace calls to load() with ref() --- models/email/emails_denormalized.sql | 2 +- models/mailchimp/mailchimp_all_events.sql | 12 ++++++------ models/mailchimp/mailchimp_bounces.sql | 4 ++-- models/mailchimp/mailchimp_clicks.sql | 2 +- models/mailchimp/mailchimp_email_summary.sql | 10 +++++----- models/mailchimp/mailchimp_opens.sql | 2 +- models/mailchimp/mailchimp_sends.sql | 4 ++-- models/pardot/pardot_emails.sql | 2 +- models/pardot/pardot_tests.sql | 2 +- models/pardot/pardot_visitoractivity.sql | 4 ++-- models/stripe/stripe_invoices_cleaned.sql | 2 +- models/stripe/stripe_invoices_transformed.sql | 6 +++--- models/trello/trello_model_tests.sql | 4 ++-- .../zuora_subscriptions_w_charges_and_amendments.sql | 10 +++++----- 14 files changed, 33 insertions(+), 33 deletions(-) diff --git a/models/email/emails_denormalized.sql b/models/email/emails_denormalized.sql index 81e6192..9d1ac73 100644 --- a/models/email/emails_denormalized.sql +++ b/models/email/emails_denormalized.sql @@ -1,6 +1,6 @@ with events as ( - select * from {{load('pardot_emails')}} + select * from {{ref('pardot_emails')}} ), diff --git a/models/mailchimp/mailchimp_all_events.sql b/models/mailchimp/mailchimp_all_events.sql index d62074c..9a22d64 100644 --- a/models/mailchimp/mailchimp_all_events.sql +++ b/models/mailchimp/mailchimp_all_events.sql @@ -1,34 +1,34 @@ with sends as ( select campaign_id, email_id, 'sent' as event_action, sent_date as event_date - from {{load('mailchimp_sends')}} + from {{ref('mailchimp_sends')}} ), soft_bounces as ( select campaign_id, email_id, 'soft bounce' as event_action, bounced_date as event_date - from {{load('mailchimp_bounces')}} + from {{ref('mailchimp_bounces')}} where bounce_type = 'soft' ), hard_bounces as ( select campaign_id, email_id, 'hard bounce' as event_action, bounced_date as event_date - from {{load('mailchimp_bounces')}} + from {{ref('mailchimp_bounces')}} where bounce_type = 'hard' ), opens as ( select campaign_id, email_id, 'opened' as event_action, opened_date as event_date - from {{load('mailchimp_opens')}} + from {{ref('mailchimp_opens')}} ), clicks as ( select campaign_id, email_id, 'clicked' as event_action, clicked_date as event_date - from {{load('mailchimp_clicks')}} + from {{ref('mailchimp_clicks')}} ), unsubscribes as ( select campaign_id, email_id, 'unsubscribed' as event_action, unsubscribed_date as event_date - from {{load('mailchimp_unsubscribes')}} + from {{ref('mailchimp_unsubscribes')}} ) select * diff --git a/models/mailchimp/mailchimp_bounces.sql b/models/mailchimp/mailchimp_bounces.sql index cad4950..80bf9d1 100644 --- a/models/mailchimp/mailchimp_bounces.sql +++ b/models/mailchimp/mailchimp_bounces.sql @@ -1,6 +1,6 @@ select a.campaign_id, a.email_id, action_date as bounced_date, status as bounce_type -from {{load('mailchimp_email_actions')}} a -inner join {{load('mailchimp_sent_to')}} b +from {{ref('mailchimp_email_actions')}} a +inner join {{ref('mailchimp_sent_to')}} b on a.campaign_id = b.campaign_id and a.email_id = b.email_id where action = 'bounce' \ No newline at end of file diff --git a/models/mailchimp/mailchimp_clicks.sql b/models/mailchimp/mailchimp_clicks.sql index 3c78a4a..bc405f7 100644 --- a/models/mailchimp/mailchimp_clicks.sql +++ b/models/mailchimp/mailchimp_clicks.sql @@ -1,3 +1,3 @@ select campaign_id, email_id, action_date as clicked_date -from {{load('mailchimp_email_actions')}} +from {{ref('mailchimp_email_actions')}} where action = 'click' \ No newline at end of file diff --git a/models/mailchimp/mailchimp_email_summary.sql b/models/mailchimp/mailchimp_email_summary.sql index 0da1d4b..ae89d2f 100644 --- a/models/mailchimp/mailchimp_email_summary.sql +++ b/models/mailchimp/mailchimp_email_summary.sql @@ -1,12 +1,12 @@ with sends as ( select campaign_id, email_id, sent_date - from {{load('mailchimp_sends')}} + from {{ref('mailchimp_sends')}} ), hard_bounces as ( select campaign_id, email_id, min(bounced_date) as hard_bounced_date - from {{load('mailchimp_bounces')}} + from {{ref('mailchimp_bounces')}} where bounce_type = 'hard' group by campaign_id, email_id ), @@ -15,7 +15,7 @@ opens as ( select campaign_id, email_id, min(opened_date) as first_opened_date, max(opened_date) as last_opened_date, count(*) as total_opens - from {{load('mailchimp_opens')}} + from {{ref('mailchimp_opens')}} group by campaign_id, email_id ), @@ -23,13 +23,13 @@ clicks as ( select campaign_id, email_id, min(clicked_date) as first_clicked_date, max(clicked_date) as last_clicked_date, count(*) as total_clicks - from {{load('mailchimp_clicks')}} + from {{ref('mailchimp_clicks')}} group by campaign_id, email_id ), unsubscribes as ( select campaign_id, email_id, unsubscribed_date - from {{load('mailchimp_unsubscribes')}} + from {{ref('mailchimp_unsubscribes')}} ) select diff --git a/models/mailchimp/mailchimp_opens.sql b/models/mailchimp/mailchimp_opens.sql index 1b8b4df..dd5667c 100644 --- a/models/mailchimp/mailchimp_opens.sql +++ b/models/mailchimp/mailchimp_opens.sql @@ -1,3 +1,3 @@ select campaign_id, email_id, action_date as opened_date -from {{load('mailchimp_email_actions')}} +from {{ref('mailchimp_email_actions')}} where action = 'open' \ No newline at end of file diff --git a/models/mailchimp/mailchimp_sends.sql b/models/mailchimp/mailchimp_sends.sql index 2261bf9..df9b1c7 100644 --- a/models/mailchimp/mailchimp_sends.sql +++ b/models/mailchimp/mailchimp_sends.sql @@ -1,6 +1,6 @@ select a.campaign_id, a.email_id, sent_date -from {{load('mailchimp_sent_to')}} a +from {{ref('mailchimp_sent_to')}} a -- add information about when the campaign was sent -inner join {{load('mailchimp_campaigns')}} b +inner join {{ref('mailchimp_campaigns')}} b on a.campaign_id = b.campaign_id and a.list_id = b.list_id \ No newline at end of file diff --git a/models/pardot/pardot_emails.sql b/models/pardot/pardot_emails.sql index 60df61c..5c4c4a7 100644 --- a/models/pardot/pardot_emails.sql +++ b/models/pardot/pardot_emails.sql @@ -5,5 +5,5 @@ data necessary to conform to the extended interface. */ select "@timestamp", "@event", "@user_id", email_id as "@email_id", details as "@subject" - from {{load('pardot_visitoractivity')}} + from {{ref('pardot_visitoractivity')}} where "@event" in ('email sent', 'email opened', 'email click') diff --git a/models/pardot/pardot_tests.sql b/models/pardot/pardot_tests.sql index 76d60ef..7f94f1f 100644 --- a/models/pardot/pardot_tests.sql +++ b/models/pardot/pardot_tests.sql @@ -2,7 +2,7 @@ select 'visitoractivity_fresher_than_one_day' as name, 'Most recent visitoractivity entry is no more than one day old' as description, max("@timestamp"::timestamp) > current_date - '1 day'::interval as result -from {{load('pardot_visitoractivity')}} +from {{ref('pardot_visitoractivity')}} diff --git a/models/pardot/pardot_visitoractivity.sql b/models/pardot/pardot_visitoractivity.sql index 940bf84..39db5a0 100644 --- a/models/pardot/pardot_visitoractivity.sql +++ b/models/pardot/pardot_visitoractivity.sql @@ -9,7 +9,7 @@ select va.* from olga_pardot.visitoractivity va - inner join {{load('pardot_visitoractivity_events_meta')}} e + inner join {{ref('pardot_visitoractivity_events_meta')}} e on va."type" = e."type" and va.type_name = e.type_name - inner join {{load('pardot_visitoractivity_types_meta')}} t + inner join {{ref('pardot_visitoractivity_types_meta')}} t on va."type" = t."type" diff --git a/models/stripe/stripe_invoices_cleaned.sql b/models/stripe/stripe_invoices_cleaned.sql index 0d07205..2a5d58d 100644 --- a/models/stripe/stripe_invoices_cleaned.sql +++ b/models/stripe/stripe_invoices_cleaned.sql @@ -9,6 +9,6 @@ select timestamp 'epoch' + period_start * interval '1 Second' as period_start, timestamp 'epoch' + period_end * interval '1 Second' as period_end from - {{load('stripe_invoices')}} + {{ref('stripe_invoices')}} diff --git a/models/stripe/stripe_invoices_transformed.sql b/models/stripe/stripe_invoices_transformed.sql index 557be7e..5bcca6e 100644 --- a/models/stripe/stripe_invoices_transformed.sql +++ b/models/stripe/stripe_invoices_transformed.sql @@ -2,7 +2,7 @@ with invoices as ( select * - from {{load('stripe_invoices_cleaned')}} + from {{ref('stripe_invoices_cleaned')}} where paid is true and forgiven is false @@ -55,7 +55,7 @@ from customer_dates d on d.date_month >= date_trunc('month', i.period_start) and d.date_month < date_trunc('month', i.period_end) and d.customer = i.customer - left outer join {{load('stripe_subscriptions')}} s on i.subscription_id = s.id - left outer join {{load('stripe_plans')}} p on s.plan_id = p.id + left outer join {{ref('stripe_subscriptions')}} s on i.subscription_id = s.id + left outer join {{ref('stripe_plans')}} p on s.plan_id = p.id diff --git a/models/trello/trello_model_tests.sql b/models/trello/trello_model_tests.sql index ad9dec6..b68a3c1 100644 --- a/models/trello/trello_model_tests.sql +++ b/models/trello/trello_model_tests.sql @@ -1,7 +1,7 @@ with null_boards_or_lists as ( select id - from {{load('trello_card_location')}} + from {{ref('trello_card_location')}} where data__board__id is null or data__list__id is null @@ -18,4 +18,4 @@ select 'fresher_than_one_day', 'Most recent entry is no more than one day old', max(date::timestamp) > current_date - '1 day'::interval -from {{load('trello_card_location')}} +from {{ref('trello_card_location')}} diff --git a/models/zuora/zuora_subscriptions_w_charges_and_amendments.sql b/models/zuora/zuora_subscriptions_w_charges_and_amendments.sql index bc8453a..d2776ef 100644 --- a/models/zuora/zuora_subscriptions_w_charges_and_amendments.sql +++ b/models/zuora/zuora_subscriptions_w_charges_and_amendments.sql @@ -5,11 +5,11 @@ with subscr_w_amendments as account_number, acc.account_id, sub.subscr_id, subscr_name, subscr_status, subscr_term_type, subscr_start, subscr_end, subscr_version, amend_id, amend_start - from {{load('zuora_account')}} acc - inner join {{load('zuora_subscription')}} sub + from {{ref('zuora_account')}} acc + inner join {{ref('zuora_subscription')}} sub on acc.account_id = sub.account_id -- add ammendments - left outer join {{load('zuora_amendment')}} amend + left outer join {{ref('zuora_amendment')}} amend on sub.subscr_id = amend.subscr_id ) @@ -21,7 +21,7 @@ select min(subscr_start) over() as first_subscr, "@mrr" as mrr from subscr_w_amendments sub -inner join {{load('zuora_rate_plan')}} rp +inner join {{ref('zuora_rate_plan')}} rp on rp.subscr_id = sub.subscr_id -inner join {{load('zuora_rate_plan_charge')}} rpc +inner join {{ref('zuora_rate_plan_charge')}} rpc on rpc.rate_plan_id = rp.rate_plan_id From 8ab283ad9816a3b8884d2f595a7e70fb9b094aa4 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Thu, 5 May 2016 16:12:25 -0400 Subject: [PATCH 79/88] add package namespace --- dbt_project.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbt_project.yml b/dbt_project.yml index 1ab3d0d..3710493 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,9 @@ #settings specifically for this models directory #config other dbt settings within ~/.dbt/profiles.yml +package: + name: 'Analyst_Collective' + version: 1.0 + source-paths: ["models"] target-path: "target" clean-targets: ["target"] From 7d097167ec351093d9dcfd6df4ebda06c54e8a25 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Thu, 5 May 2016 17:02:17 -0400 Subject: [PATCH 80/88] make version a string --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index 3710493..b6c288a 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -2,7 +2,7 @@ #config other dbt settings within ~/.dbt/profiles.yml package: name: 'Analyst_Collective' - version: 1.0 + version: '1.0' source-paths: ["models"] target-path: "target" From 1c945f6f6ba5d1a6694d1be3455f688ab53d573f Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Fri, 6 May 2016 14:04:04 -0400 Subject: [PATCH 81/88] make name and version top-level --- dbt_project.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index b6c288a..24521e7 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,8 +1,7 @@ #settings specifically for this models directory #config other dbt settings within ~/.dbt/profiles.yml -package: - name: 'Analyst_Collective' - version: '1.0' +name: 'Analyst_Collective' +version: '1.0' source-paths: ["models"] target-path: "target" From dc67d37255e45aedb20048227d2b30b07b096aee Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Thu, 30 Jun 2016 11:51:58 -0400 Subject: [PATCH 82/88] moved all of these to their own individual repos --- README.md | 6 +- analysis/mailchimp/mailchimp_best_times.sql | 23 ---- .../mailchimp/mailchimp_email_timeseries.sql | 15 --- analysis/mailchimp/mailchimp_gains_losses.sql | 71 ------------ analysis/mailchimp/mailchimp_most_engaged.sql | 13 --- analysis/stripe/analysis.sql | 102 ------------------ models/mailchimp/mailchimp_all_events.sql | 52 --------- models/mailchimp/mailchimp_bounces.sql | 6 -- models/mailchimp/mailchimp_campaigns.sql | 3 - models/mailchimp/mailchimp_clicks.sql | 3 - models/mailchimp/mailchimp_email_actions.sql | 3 - models/mailchimp/mailchimp_email_summary.sql | 56 ---------- models/mailchimp/mailchimp_lists.sql | 3 - models/mailchimp/mailchimp_members.sql | 3 - models/mailchimp/mailchimp_opens.sql | 3 - models/mailchimp/mailchimp_sends.sql | 6 -- models/mailchimp/mailchimp_sent_to.sql | 3 - models/mailchimp/mailchimp_unsubscribes.sql | 2 - models/stripe/stripe_invoices.sql | 13 --- models/stripe/stripe_invoices_cleaned.sql | 14 --- models/stripe/stripe_invoices_transformed.sql | 61 ----------- models/stripe/stripe_plans.sql | 8 -- models/stripe/stripe_subscriptions.sql | 8 -- 23 files changed, 5 insertions(+), 472 deletions(-) delete mode 100644 analysis/mailchimp/mailchimp_best_times.sql delete mode 100644 analysis/mailchimp/mailchimp_email_timeseries.sql delete mode 100644 analysis/mailchimp/mailchimp_gains_losses.sql delete mode 100644 analysis/mailchimp/mailchimp_most_engaged.sql delete mode 100644 analysis/stripe/analysis.sql delete mode 100644 models/mailchimp/mailchimp_all_events.sql delete mode 100644 models/mailchimp/mailchimp_bounces.sql delete mode 100644 models/mailchimp/mailchimp_campaigns.sql delete mode 100644 models/mailchimp/mailchimp_clicks.sql delete mode 100644 models/mailchimp/mailchimp_email_actions.sql delete mode 100644 models/mailchimp/mailchimp_email_summary.sql delete mode 100644 models/mailchimp/mailchimp_lists.sql delete mode 100644 models/mailchimp/mailchimp_members.sql delete mode 100644 models/mailchimp/mailchimp_opens.sql delete mode 100644 models/mailchimp/mailchimp_sends.sql delete mode 100644 models/mailchimp/mailchimp_sent_to.sql delete mode 100644 models/mailchimp/mailchimp_unsubscribes.sql delete mode 100644 models/stripe/stripe_invoices.sql delete mode 100644 models/stripe/stripe_invoices_cleaned.sql delete mode 100644 models/stripe/stripe_invoices_transformed.sql delete mode 100644 models/stripe/stripe_plans.sql delete mode 100644 models/stripe/stripe_subscriptions.sql diff --git a/README.md b/README.md index 714f86a..9e729b5 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ This repository contains two primary types of objects: data models and data anal - Models can be configured in dbt to be materialized as either views or tables. - Model files should go into `/models` and saved with a `.sql` extension. - Each model should be stored in its own `.sql` file. The file name will become the name of the table or view in the database. -- Environment configuration should be supplied via `{{env}}`. +- Other models should be referenced with the `ref` function. This function will resolve dependencies during the `compile` stage. The only tables referenced without this function should be source raw data tables. - Models should be designed to minimize the selection from raw data tables. This minimizes the amount of mapping end users of models will need to do when configuring them for their local environment. ##### Analysis @@ -22,6 +22,10 @@ This repository contains two primary types of objects: data models and data anal - All named fields in a given analysis should be named within a given model. - Confining analysis in this way ensures portability of analysis across multiple environments. +##### Dependencies + +- All projects can include dependencies to other projects. Dependencies + ### Contributing All contributions to this repository must be for analytics on top of standardized datasets. The current process for contributing is to: diff --git a/analysis/mailchimp/mailchimp_best_times.sql b/analysis/mailchimp/mailchimp_best_times.sql deleted file mode 100644 index 579a1e6..0000000 --- a/analysis/mailchimp/mailchimp_best_times.sql +++ /dev/null @@ -1,23 +0,0 @@ -with email_summary as ( - - select * - from {{env.schema}}.mailchimp_email_summary - -), best_day_of_week as ( - - select date_part(dow, sent_date), avg(opened::float) - from email_summary - group by 1 - order by 1 - -), best_hour_of_day as ( - - select date_part(hr, sent_date), avg(opened::float) - from email_summary - group by 1 - order by 1 - -) - -select * -from best_day_of_week diff --git a/analysis/mailchimp/mailchimp_email_timeseries.sql b/analysis/mailchimp/mailchimp_email_timeseries.sql deleted file mode 100644 index 0230e86..0000000 --- a/analysis/mailchimp/mailchimp_email_timeseries.sql +++ /dev/null @@ -1,15 +0,0 @@ -with events as ( - select * - from {{env.schema}}.mailchimp_email_summary -) - -select - date_trunc('month', sent_date) as month, - count(*) as sends, - sum(opened) as opens, - sum(clicked) as clicks, - avg(opened::float) as open_rate, - avg(clicked::float) as click_rate -from events -group by month -order by month diff --git a/analysis/mailchimp/mailchimp_gains_losses.sql b/analysis/mailchimp/mailchimp_gains_losses.sql deleted file mode 100644 index 21e7d95..0000000 --- a/analysis/mailchimp/mailchimp_gains_losses.sql +++ /dev/null @@ -1,71 +0,0 @@ -with -gains as ( - select date_trunc('month', signup_date) as month, count(*) total_gains - from {{env.schema}}.mailchimp_members - group by date_trunc('month', signup_date) -), - -hard_bounces as -( - -- get the first hard bounce date for each email - select email_id, min(bounced_date) as event_date - from {{env.schema}}.mailchimp_bounces - where bounce_type = 'hard' - group by email_id -), - -unsubscribes as -( - -- get the first unsubscribed date for each email - select email_id, min(unsubscribed_date) as event_date - from {{env.schema}}.mailchimp_unsubscribes - group by email_id -), - -losses as ( - -- count losses for each month - select date_trunc('month', event_date) as month, count(*) as total_losses - from - ( - -- select the first of unsubscribe or hard bounce - select email_id, min(event_date) as event_date - from - ( - select email_id, event_date - from hard_bounces - union - select email_id, event_date - from unsubscribes - ) - group by email_id - ) - group by month -), - -months as ( - select month - from gains - union - select month - from losses -), - -monthly_gains_and_losses as ( - - select - months.month, nvl(total_gains,0) as month_gains, - -nvl(total_losses,0) as month_losses - from months - left outer join gains - on months.month = gains.month - left outer join losses - on months.month = losses.month - order by months.month - -) - -select month, month_gains, month_losses, month_gains + month_losses as net_member_growth, - sum(month_gains) over (order by month rows unbounded preceding) as cum_gains, - sum(month_losses) over (order by month rows unbounded preceding) as cum_losses, - sum(month_gains + month_losses) over (order by month rows unbounded preceding) as active_members -from monthly_gains_and_losses diff --git a/analysis/mailchimp/mailchimp_most_engaged.sql b/analysis/mailchimp/mailchimp_most_engaged.sql deleted file mode 100644 index 05a7534..0000000 --- a/analysis/mailchimp/mailchimp_most_engaged.sql +++ /dev/null @@ -1,13 +0,0 @@ -with email_summary as ( - - select * - from {{env.schema}}.mailchimp_email_summary - -) - -select email_id, sum(clicked)::float / sum(opened) -from email_summary -group by 1 -having sum(opened) > 5 -order by 2 desc -limit 100 diff --git a/analysis/stripe/analysis.sql b/analysis/stripe/analysis.sql deleted file mode 100644 index a6630e3..0000000 --- a/analysis/stripe/analysis.sql +++ /dev/null @@ -1,102 +0,0 @@ -with invoices as ( - - select * - from {{env.schema}}.stripe_invoices_transformed - -), all_months as ( - - select distinct date_month from invoices - -), plan_changes as ( - - select - *, - lag(total) over (partition by customer order by date_month) as prior_month_total, - total - coalesce(lag(total) over (partition by customer order by date_month), 0) as change, - lag(period_end) over (partition by customer order by date_month) as prior_month_period_end - from invoices - -), data as ( - - select *, - case - when first_payment = 1 - then 'new' - when last_payment = 1 - and dateadd('month', 1, period_end) < current_date - then 'churn' - when change > 0 - then 'upgrade' - when change < 0 - then 'downgrade' - when period != 'monthly' - and date_month < date_trunc('month', prior_month_period_end) - then 'prepaid renewal' - else - 'renewal' - end revenue_category, - case - when prior_month_total < total then prior_month_total - else total - end renewal_component_of_change - from plan_changes - -), news as ( - - select date_month, sum(total) as value - from data - where revenue_category = 'new' - group by 1 - -), renewals as ( - - select date_month, sum(renewal_component_of_change) as value - from data - where revenue_category in ('renewal', 'downgrade', 'upgrade') - group by 1 - -), prepaids as ( - - select date_month, sum(total) as value - from data - where revenue_category = 'prepaid renewal' - group by 1 - -), churns as ( - - select date_month, sum(total) as value - from data - where revenue_category = 'churn' - group by 1 - -), upgrades as ( - - select date_month, sum(change) as value - from data - where revenue_category = 'upgrade' - group by 1 - -), downgrades as ( - - select date_month, sum(change) as value - from data - where revenue_category = 'downgrade' - group by 1 - -) - -select all_months.date_month, - news.value as new, - renewals.value as renewal, - prepaids.value as committed, - churns.value * -1 as churned, - upgrades.value as upgrades, - downgrades.value as downgrades -from all_months - left outer join news on all_months.date_month = news.date_month - left outer join renewals on all_months.date_month = renewals.date_month - left outer join prepaids on all_months.date_month = prepaids.date_month - left outer join churns on all_months.date_month = churns.date_month - left outer join upgrades on all_months.date_month = upgrades.date_month - left outer join downgrades on all_months.date_month = downgrades.date_month -order by 1 diff --git a/models/mailchimp/mailchimp_all_events.sql b/models/mailchimp/mailchimp_all_events.sql deleted file mode 100644 index 9a22d64..0000000 --- a/models/mailchimp/mailchimp_all_events.sql +++ /dev/null @@ -1,52 +0,0 @@ -with -sends as ( - select campaign_id, email_id, 'sent' as event_action, sent_date as event_date - from {{ref('mailchimp_sends')}} -), - -soft_bounces as ( - select campaign_id, email_id, 'soft bounce' as event_action, bounced_date as event_date - from {{ref('mailchimp_bounces')}} - where bounce_type = 'soft' -), - -hard_bounces as ( - select campaign_id, email_id, 'hard bounce' as event_action, bounced_date as event_date - from {{ref('mailchimp_bounces')}} - where bounce_type = 'hard' -), - -opens as ( - select campaign_id, email_id, 'opened' as event_action, opened_date as event_date - from {{ref('mailchimp_opens')}} -), - -clicks as ( - select campaign_id, email_id, 'clicked' as event_action, clicked_date as event_date - from {{ref('mailchimp_clicks')}} -), - -unsubscribes as ( - select campaign_id, email_id, 'unsubscribed' as event_action, unsubscribed_date as event_date - from {{ref('mailchimp_unsubscribes')}} -) - -select * -from sends -union -select * -from soft_bounces -union -select * -from hard_bounces -union -select * -from opens -union -select * -from clicks -union -select * -from unsubscribes - - diff --git a/models/mailchimp/mailchimp_bounces.sql b/models/mailchimp/mailchimp_bounces.sql deleted file mode 100644 index 80bf9d1..0000000 --- a/models/mailchimp/mailchimp_bounces.sql +++ /dev/null @@ -1,6 +0,0 @@ -select a.campaign_id, a.email_id, action_date as bounced_date, status as bounce_type -from {{ref('mailchimp_email_actions')}} a -inner join {{ref('mailchimp_sent_to')}} b - on a.campaign_id = b.campaign_id - and a.email_id = b.email_id -where action = 'bounce' \ No newline at end of file diff --git a/models/mailchimp/mailchimp_campaigns.sql b/models/mailchimp/mailchimp_campaigns.sql deleted file mode 100644 index 76ed116..0000000 --- a/models/mailchimp/mailchimp_campaigns.sql +++ /dev/null @@ -1,3 +0,0 @@ -select - recipients__list_id as list_id, id as campaign_id, send_time as sent_date -from demo_data.mailchimp_campaigns \ No newline at end of file diff --git a/models/mailchimp/mailchimp_clicks.sql b/models/mailchimp/mailchimp_clicks.sql deleted file mode 100644 index bc405f7..0000000 --- a/models/mailchimp/mailchimp_clicks.sql +++ /dev/null @@ -1,3 +0,0 @@ -select campaign_id, email_id, action_date as clicked_date -from {{ref('mailchimp_email_actions')}} -where action = 'click' \ No newline at end of file diff --git a/models/mailchimp/mailchimp_email_actions.sql b/models/mailchimp/mailchimp_email_actions.sql deleted file mode 100644 index e322893..0000000 --- a/models/mailchimp/mailchimp_email_actions.sql +++ /dev/null @@ -1,3 +0,0 @@ -select - campaign_id, email_id, action, "timestamp" as action_date -from demo_data.mailchimp_email_activity__activity \ No newline at end of file diff --git a/models/mailchimp/mailchimp_email_summary.sql b/models/mailchimp/mailchimp_email_summary.sql deleted file mode 100644 index ae89d2f..0000000 --- a/models/mailchimp/mailchimp_email_summary.sql +++ /dev/null @@ -1,56 +0,0 @@ -with -sends as ( - select campaign_id, email_id, sent_date - from {{ref('mailchimp_sends')}} -), - -hard_bounces as ( - select campaign_id, email_id, min(bounced_date) as hard_bounced_date - from {{ref('mailchimp_bounces')}} - where bounce_type = 'hard' - group by campaign_id, email_id -), - -opens as ( - select - campaign_id, email_id, min(opened_date) as first_opened_date, - max(opened_date) as last_opened_date, count(*) as total_opens - from {{ref('mailchimp_opens')}} - group by campaign_id, email_id -), - -clicks as ( - select - campaign_id, email_id, min(clicked_date) as first_clicked_date, - max(clicked_date) as last_clicked_date, count(*) as total_clicks - from {{ref('mailchimp_clicks')}} - group by campaign_id, email_id -), - -unsubscribes as ( - select campaign_id, email_id, unsubscribed_date - from {{ref('mailchimp_unsubscribes')}} -) - -select - s.campaign_id, s.email_id, sent_date, hard_bounced_date, first_opened_date, - last_opened_date, total_opens, first_clicked_date, last_clicked_date, - total_clicks, unsubscribed_date, - decode(hard_bounced_date, null, 0, 1) as hard_bounced, - decode(first_opened_date, null, 0, 1) as opened, - decode(first_clicked_date, null, 0, 1) as clicked, - decode(unsubscribed_date, null, 0, 1) as unsubscribed -from sends s -left outer join hard_bounces b - on s.email_id = b.email_id and - s.campaign_id = b.campaign_id -left outer join opens o - on s.email_id = o.email_id and - s.campaign_id = o.campaign_id -left outer join clicks c - on s.email_id = c.email_id and - s.campaign_id = c.campaign_id -left outer join unsubscribes u - on s.email_id = u.email_id and - s.campaign_id = u.campaign_id -order by email_id, sent_date diff --git a/models/mailchimp/mailchimp_lists.sql b/models/mailchimp/mailchimp_lists.sql deleted file mode 100644 index 6762eed..0000000 --- a/models/mailchimp/mailchimp_lists.sql +++ /dev/null @@ -1,3 +0,0 @@ -select - id as list_id, name as list_name -from demo_data.mailchimp_lists \ No newline at end of file diff --git a/models/mailchimp/mailchimp_members.sql b/models/mailchimp/mailchimp_members.sql deleted file mode 100644 index 927513e..0000000 --- a/models/mailchimp/mailchimp_members.sql +++ /dev/null @@ -1,3 +0,0 @@ -select - list_id, "timestamp_signup" as signup_date, id as email_id, email_address -from demo_data.mailchimp_members \ No newline at end of file diff --git a/models/mailchimp/mailchimp_opens.sql b/models/mailchimp/mailchimp_opens.sql deleted file mode 100644 index dd5667c..0000000 --- a/models/mailchimp/mailchimp_opens.sql +++ /dev/null @@ -1,3 +0,0 @@ -select campaign_id, email_id, action_date as opened_date -from {{ref('mailchimp_email_actions')}} -where action = 'open' \ No newline at end of file diff --git a/models/mailchimp/mailchimp_sends.sql b/models/mailchimp/mailchimp_sends.sql deleted file mode 100644 index df9b1c7..0000000 --- a/models/mailchimp/mailchimp_sends.sql +++ /dev/null @@ -1,6 +0,0 @@ -select a.campaign_id, a.email_id, sent_date -from {{ref('mailchimp_sent_to')}} a --- add information about when the campaign was sent -inner join {{ref('mailchimp_campaigns')}} b - on a.campaign_id = b.campaign_id - and a.list_id = b.list_id \ No newline at end of file diff --git a/models/mailchimp/mailchimp_sent_to.sql b/models/mailchimp/mailchimp_sent_to.sql deleted file mode 100644 index 6207c37..0000000 --- a/models/mailchimp/mailchimp_sent_to.sql +++ /dev/null @@ -1,3 +0,0 @@ -select - list_id, campaign_id, email_id, email_address, status -from demo_data.mailchimp_sent_to \ No newline at end of file diff --git a/models/mailchimp/mailchimp_unsubscribes.sql b/models/mailchimp/mailchimp_unsubscribes.sql deleted file mode 100644 index 3de8453..0000000 --- a/models/mailchimp/mailchimp_unsubscribes.sql +++ /dev/null @@ -1,2 +0,0 @@ -select campaign_id, email_id, "timestamp" as unsubscribed_date -from demo_data.mailchimp_unsubscribes \ No newline at end of file diff --git a/models/stripe/stripe_invoices.sql b/models/stripe/stripe_invoices.sql deleted file mode 100644 index 1e3c89c..0000000 --- a/models/stripe/stripe_invoices.sql +++ /dev/null @@ -1,13 +0,0 @@ - -select - customer, - date, - forgiven, - subscription as subscription_id, - paid, - total, - period_start, - period_end -from - demo_data.stripe_invoices - diff --git a/models/stripe/stripe_invoices_cleaned.sql b/models/stripe/stripe_invoices_cleaned.sql deleted file mode 100644 index 2a5d58d..0000000 --- a/models/stripe/stripe_invoices_cleaned.sql +++ /dev/null @@ -1,14 +0,0 @@ - -select - customer, - timestamp 'epoch' + date * interval '1 Second' as date, - forgiven, - subscription_id, - paid, - total, - timestamp 'epoch' + period_start * interval '1 Second' as period_start, - timestamp 'epoch' + period_end * interval '1 Second' as period_end -from - {{ref('stripe_invoices')}} - - diff --git a/models/stripe/stripe_invoices_transformed.sql b/models/stripe/stripe_invoices_transformed.sql deleted file mode 100644 index 5bcca6e..0000000 --- a/models/stripe/stripe_invoices_transformed.sql +++ /dev/null @@ -1,61 +0,0 @@ - -with invoices as ( - - select * - from {{ref('stripe_invoices_cleaned')}} - where paid is true - and forgiven is false - -), days as ( - - select (min(period_start) over () + row_number() over ())::date as date_day - from invoices - -), months as ( - - select distinct date_trunc('month', date_day)::date as date_month - from days - where date_day <= current_date - -), customers as ( - - select customer, min(period_start) as active_from, max(period_end) as active_to - from invoices - where period_start <= current_date - group by customer - -), customer_dates as ( - - select m.date_month, c.customer - from months m - inner join customers c - on m.date_month >= date_trunc('month', c.active_from) - and m.date_month < date_trunc('month', c.active_to) - -) - -select date_month, d.customer, period_start, period_end, - "interval" as period, - case "interval" - when 'yearly' - then coalesce(i.total, 0)::float / 12 / 100 - else - coalesce(i.total, 0)::float / 100 - end as total, - case min(date_month) over(partition by d.customer) - when date_month then 1 - else 0 - end as first_payment, - case max(date_month) over(partition by d.customer) - when date_month then 1 - else 0 - end as last_payment -from customer_dates d - left outer join invoices i - on d.date_month >= date_trunc('month', i.period_start) - and d.date_month < date_trunc('month', i.period_end) - and d.customer = i.customer - left outer join {{ref('stripe_subscriptions')}} s on i.subscription_id = s.id - left outer join {{ref('stripe_plans')}} p on s.plan_id = p.id - - diff --git a/models/stripe/stripe_plans.sql b/models/stripe/stripe_plans.sql deleted file mode 100644 index 6600368..0000000 --- a/models/stripe/stripe_plans.sql +++ /dev/null @@ -1,8 +0,0 @@ - -select - id, - interval -from - demo_data.stripe_plans - - diff --git a/models/stripe/stripe_subscriptions.sql b/models/stripe/stripe_subscriptions.sql deleted file mode 100644 index ce94ba0..0000000 --- a/models/stripe/stripe_subscriptions.sql +++ /dev/null @@ -1,8 +0,0 @@ - -select - id, - plan__id as plan_id -from - demo_data.stripe_subscriptions - - From bcbf61ab410952dcfc0d8fc04a017eaa1fa36e70 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Mon, 1 Aug 2016 16:35:22 -0400 Subject: [PATCH 83/88] clean up old code --- models/email/emails_denormalized.sql | 36 ------------------ models/magento/order_items.sql | 3 -- models/magento/products.sql | 4 -- models/pardot/pardot_emails.sql | 9 ----- models/pardot/pardot_tests.sql | 20 ---------- models/pardot/pardot_visitoractivity.sql | 15 -------- .../pardot_visitoractivity_events_meta.sql | 35 ------------------ .../pardot_visitoractivity_types_meta.sql | 37 ------------------- models/segment/segment_track.sql | 8 ---- models/snowplow/snowplow_events.sql | 7 ---- 10 files changed, 174 deletions(-) delete mode 100644 models/email/emails_denormalized.sql delete mode 100644 models/magento/order_items.sql delete mode 100644 models/magento/products.sql delete mode 100644 models/pardot/pardot_emails.sql delete mode 100644 models/pardot/pardot_tests.sql delete mode 100644 models/pardot/pardot_visitoractivity.sql delete mode 100644 models/pardot/pardot_visitoractivity_events_meta.sql delete mode 100644 models/pardot/pardot_visitoractivity_types_meta.sql delete mode 100644 models/segment/segment_track.sql delete mode 100644 models/snowplow/snowplow_events.sql diff --git a/models/email/emails_denormalized.sql b/models/email/emails_denormalized.sql deleted file mode 100644 index 9d1ac73..0000000 --- a/models/email/emails_denormalized.sql +++ /dev/null @@ -1,36 +0,0 @@ -with events as ( - - select * from {{ref('pardot_emails')}} - -), - -sends as ( - - select "@user_id", "@timestamp", "@email_id" - from events - where "@event" = 'email sent' - -), - -opens as ( - - select "@user_id", "@timestamp", "@email_id" - from events - where "@event" = 'email opened' - -), clicks as ( - - select "@user_id", "@timestamp", "@email_id" - from events - where "@event" = 'email click' - -) - -select s."@user_id", s."@timestamp" as sent_timestamp, o."@timestamp" as opened_timestamp, c."@timestamp" as clicked_timestamp, - decode(o."@timestamp", null, 0, 1) as "opened?", - decode(c."@timestamp", null, 0, 1) as "clicked?", - row_number() over (partition by s."@user_id" order by s."@timestamp") as email_number -from sends s - left outer join opens o on s."@email_id" = o."@email_id" - left outer join clicks c on s."@email_id" = c."@email_id" -order by 1, 2 diff --git a/models/magento/order_items.sql b/models/magento/order_items.sql deleted file mode 100644 index 07d33d3..0000000 --- a/models/magento/order_items.sql +++ /dev/null @@ -1,3 +0,0 @@ -select * -FROM - sample_magento_database.sales_flat_order_item diff --git a/models/magento/products.sql b/models/magento/products.sql deleted file mode 100644 index 87468fb..0000000 --- a/models/magento/products.sql +++ /dev/null @@ -1,4 +0,0 @@ -select * -FROM - sample_magento_database.catalog_product_flat_1 - diff --git a/models/pardot/pardot_emails.sql b/models/pardot/pardot_emails.sql deleted file mode 100644 index 5c4c4a7..0000000 --- a/models/pardot/pardot_emails.sql +++ /dev/null @@ -1,9 +0,0 @@ -/* -This model maps pardot data from the visitoractivity table to the email analysis interface. -It conforms to the basic email interface, not the extended email interface, because Pardot does not supply -data necessary to conform to the extended interface. -*/ - -select "@timestamp", "@event", "@user_id", email_id as "@email_id", details as "@subject" - from {{ref('pardot_visitoractivity')}} -where "@event" in ('email sent', 'email opened', 'email click') diff --git a/models/pardot/pardot_tests.sql b/models/pardot/pardot_tests.sql deleted file mode 100644 index 7f94f1f..0000000 --- a/models/pardot/pardot_tests.sql +++ /dev/null @@ -1,20 +0,0 @@ -select - 'visitoractivity_fresher_than_one_day' as name, - 'Most recent visitoractivity entry is no more than one day old' as description, - max("@timestamp"::timestamp) > current_date - '1 day'::interval as result -from {{ref('pardot_visitoractivity')}} - - - - - - - -/* - -Other tests I want to do: -- make sure there are records from every day since the first day we see any records -- make sure all prospect ids from visitoractivity show up in prospects -- make sure there are no unmapped types - -*/ diff --git a/models/pardot/pardot_visitoractivity.sql b/models/pardot/pardot_visitoractivity.sql deleted file mode 100644 index 39db5a0..0000000 --- a/models/pardot/pardot_visitoractivity.sql +++ /dev/null @@ -1,15 +0,0 @@ ---this table has a bunch of types that really should be event actions but are very poorly formulated. ---the custom logic in this view is an attempt to fix that. ---not all of the various type / type_name combinations have been accounted for yet; I still need to determine exactly what some of them mean. -select - -- event_stream interface - va.created_at as "@timestamp", - e.event_name as "@event", - va.prospect_id as "@user_id", - va.* -from - olga_pardot.visitoractivity va - inner join {{ref('pardot_visitoractivity_events_meta')}} e - on va."type" = e."type" and va.type_name = e.type_name - inner join {{ref('pardot_visitoractivity_types_meta')}} t - on va."type" = t."type" diff --git a/models/pardot/pardot_visitoractivity_events_meta.sql b/models/pardot/pardot_visitoractivity_events_meta.sql deleted file mode 100644 index 2339b65..0000000 --- a/models/pardot/pardot_visitoractivity_events_meta.sql +++ /dev/null @@ -1,35 +0,0 @@ ---even with the type decoding that Pardot specifically provides, actually what is going on in a given event ---is somewhat ambiguous. this is an attempt to map type and type_name to a more event-based "event action" field ---which is always written in more standard action-oriented terms. -select 22 as "type", 'Chat Transcript' as type_name, 'chatted via olark' as event_name union all -select 21, 'Custom Redirect', 'clicked a custom redirect' union all -select 6, 'Email', 'email sent' union all -select 11, 'Email', 'email opened' union all -select 13, 'Email', 'email bounced' union all -select 14, 'Email', 'email reported spam' union all -select 1, 'Email Tracker', 'email click' union all -select 28, 'Event', 'registered for event' union all -select 29, 'Event', 'checked in at event' union all -select 2, 'File', 'viewed a file' union all -select 3, 'Form', 'submitted a form with an error' union all -select 2, 'Form', 'viewed a form' union all -select 4, 'Form', 'successfully submitted a form' union all -select 4, 'Form Handler', 'successfully submitted a form handler' union all -select 2, 'Landing Page', 'viewed a landing page' union all -select 4, 'Landing Page', 'successfully submitted the form on a landing page' union all -select 3, 'Landing Page', 'submitted the form on a landing page with an error' union all -select 2, 'Multivariate Landing Page', 'viewed multivariate landing page' union all -select 4, 'Multivariate Landing Page', 'successfully submitted multivariate landing page' union all -select 3, 'Multivariate Landing Page', 'submitted multivariate landing page with an error' union all -select 8, 'New Opportunity', 'opened opportunity' union all -select 19, 'Opportunity Associated', 'linked existing opportunity' union all -select 10, 'Opportunity Lost', 'lost opportunity' union all -select 9, 'Opportunity Won', 'won opportunity' union all -select 2, 'Page View', 'viewed highlighted page' union all -select 34, 'Video', 'watched 75% or more of video' union all -select 27, 'Video', 'watched video' union all -select 30, 'Video', 'converted from video call to action' union all -select 20, 'Visit', 'visited website' union all -select 25, 'Webinar', 'registered for webinar' union all -select 24, 'Webinar', 'attended webinar' union all -select 18, '', 'reopened opportunity' diff --git a/models/pardot/pardot_visitoractivity_types_meta.sql b/models/pardot/pardot_visitoractivity_types_meta.sql deleted file mode 100644 index 0e664bf..0000000 --- a/models/pardot/pardot_visitoractivity_types_meta.sql +++ /dev/null @@ -1,37 +0,0 @@ ---these literal values are pulled from pardot's api docs here: ---http://developer.pardot.com/kb/object-field-references/#visitor-activity ---they change periodically over time and this query will need to be correspondingly modified. -select 1 as type, 'Click' as type_decoded union all -select 2, 'View' union all -select 3, 'Error' union all -select 4, 'Success' union all -select 5, 'Session' union all -select 6, 'Sent' union all -select 7, 'Search' union all -select 8, 'New Opportunity' union all -select 9, 'Opportunity Won' union all -select 10, 'Opportunity Lost' union all -select 11, 'Open' union all -select 12, 'Unsubscribe Page' union all -select 13, 'Bounced' union all -select 14, 'Spam Complaint' union all -select 15, 'Email Preference Page' union all -select 16, 'Resubscribed' union all -select 17, 'Click (Third Party)' union all -select 18, 'Opportunity Reopened' union all -select 19, 'Opportunity Linked' union all -select 20, 'Visit' union all -select 21, 'Custom URL click' union all -select 22, 'Olark Chat' union all -select 23, 'Invited to Webinar' union all -select 24, 'Attended Webinar' union all -select 25, 'Registered for Webinar' union all -select 26, 'Social Post Click' union all -select 27, 'Video View' union all -select 28, 'Event Registered' union all -select 29, 'Event Checked In' union all -select 30, 'Video Conversion' union all -select 31, 'UserVoice Suggestion' union all -select 32, 'UserVoice Comment' union all -select 33, 'UserVoice Ticket' union all -select 34, 'Video Watched (>= 75% watched)' diff --git a/models/segment/segment_track.sql b/models/segment/segment_track.sql deleted file mode 100644 index b393fea..0000000 --- a/models/segment/segment_track.sql +++ /dev/null @@ -1,8 +0,0 @@ -select - "timestamp"::timestamp as "@timestamp", - "event" as "@event", - "userid" as "@user_id", - * - -from - segment.track diff --git a/models/snowplow/snowplow_events.sql b/models/snowplow/snowplow_events.sql deleted file mode 100644 index f438b35..0000000 --- a/models/snowplow/snowplow_events.sql +++ /dev/null @@ -1,7 +0,0 @@ -select - "collector_tstamp" as "@timestamp", - "event_name" as "@event", - "domain_userid" as "@user_id", - * -from - atomic.events From 0f9d14f7e56ec174fa2706ce3920f935f46ce314 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Mon, 1 Aug 2016 16:37:33 -0400 Subject: [PATCH 84/88] clean up old code --- analysis/email/analysis.sql | 88 ---------------------------- analysis/email/interface.txt | 22 ------- analysis/event_stream/funnel.sql | 42 ------------- analysis/event_stream/interface.txt | 9 --- analysis/event_stream/timeseries.sql | 8 --- analysis/magento/analysis.sql | 42 ------------- 6 files changed, 211 deletions(-) delete mode 100644 analysis/email/analysis.sql delete mode 100644 analysis/email/interface.txt delete mode 100644 analysis/event_stream/funnel.sql delete mode 100644 analysis/event_stream/interface.txt delete mode 100644 analysis/event_stream/timeseries.sql delete mode 100644 analysis/magento/analysis.sql diff --git a/analysis/email/analysis.sql b/analysis/email/analysis.sql deleted file mode 100644 index 08f3ef8..0000000 --- a/analysis/email/analysis.sql +++ /dev/null @@ -1,88 +0,0 @@ -/* - -This analysis gets you a simple timeseries of sent > opened > clicked emails for all-time. - -*/ - - -with events as ( - - select * from {{env.schema}}.emails_denormalized - -) - -select - date_trunc('month', "sent_timestamp") as mnth, - count(*) as sends, - sum("opened?") as opens, - sum("clicked?") as clicks, - avg("opened?"::float) as open_rate, - avg("clicked?"::float) as click_rate -from events -group by 1 -order by 1 -; - - - -/* - -Open/click rate by user's email number. Email number 1 is the first email sent to that user, etc. -Do people stop engaging with emails the more emails they have been sent? - -*/ - - -with emails as ( - - select * from {{env.schema}}.emails_denormalized - -) - -select email_number, avg("opened?"::float) as open_rate, avg("clicked?"::float) as ctr, - count(*) as num_users -from emails ---only look at the first 25 emails someone is sent. should be customized based on business. -where email_number < 26 -group by 1 -order by 1 -; - - - - -/* - -Open/Click rate by email frequency. -Does email sending volume have an impact on engagement with emails sent? -In the extended email interface, need to do the same thing for unsubscribes, because this is where we have seen the strong correlation in the past. -This interface doesn't contain unsubscribe data. - -*/ - - -with emails as ( - - select * from {{env.schema}}.emails_denormalized - -) - -select date_trunc('month', sent_timestamp), - count(*)::float / count(distinct "@user_id")::float as emails_per_user, - avg("opened?"::float) as open_rate, avg("clicked?"::float) as ctr -from emails -group by 1 -order by 1 -; - - - - -/* - -Other analysis I still want to do: - - cohort analysis of email engagement vs first email send date. - - anomaly detection for email performance: particularly good or bad email subject lines by open rate. leave out bottom 20% of send volume. - - likelihood of opening / clicking by length (time or # of emails) of user inactivity - -*/ diff --git a/analysis/email/interface.txt b/analysis/email/interface.txt deleted file mode 100644 index dea0bed..0000000 --- a/analysis/email/interface.txt +++ /dev/null @@ -1,22 +0,0 @@ -######################################## -# Email Interface # -######################################## - -timestamp timestamp -event varchar -user_id varchar -email_id varchar -details varchar - - -The following events are need for the basic email interface: -- send -- open -- click - -The following events are need for the extended email interface: -- hard bounce -- soft bounce -- subscribe -- unsubscribe -- spam complaint diff --git a/analysis/event_stream/funnel.sql b/analysis/event_stream/funnel.sql deleted file mode 100644 index d2b8695..0000000 --- a/analysis/event_stream/funnel.sql +++ /dev/null @@ -1,42 +0,0 @@ -WITH -source as ( - select * from {{env.schema}}.snowplow_events -- change this view for your analysis -), -step_1 as ( - SELECT MIN("@timestamp") as "@timestamp", "@user_id" - FROM source - WHERE "@event" = 'page_view' -- filter by whichever columns you need - GROUP BY "@user_id" -), --- add more steps as you need. If you do add more steps, make sure to add a join below -step_2 as ( - SELECT MIN("@timestamp") as "@timestamp", "@user_id" - FROM source - WHERE "@event" = 'page_ping' - GROUP BY "@user_id" -), -step_3 as ( - SELECT MIN("@timestamp") as "@timestamp", "@user_id" - FROM source - WHERE "@event" = 'link_click' - GROUP BY "@user_id" -), -funnel as ( - --where the magic happens! - SELECT step_1."@user_id" as "step_1_users", - step_2."@user_id" as "step_2_users", - step_3."@user_id" as "step_3_users" - from step_1 - -- add more joins to make funnels with more steps - LEFT OUTER JOIN step_2 ON step_1."@user_id" = step_2."@user_id" and step_1."@timestamp" < step_2."@timestamp" - LEFT OUTER JOIN step_3 ON step_2."@user_id" = step_3."@user_id" and step_2."@timestamp" < step_3."@timestamp" - -- filter by time here - where step_1."@timestamp" > getdate() - interval '1 week' -) - -select -count(distinct step_1_users) as step_1, -count(distinct step_2_users) as step_2, -count(distinct step_3_users) as step_3 - -from funnel diff --git a/analysis/event_stream/interface.txt b/analysis/event_stream/interface.txt deleted file mode 100644 index a2d0144..0000000 --- a/analysis/event_stream/interface.txt +++ /dev/null @@ -1,9 +0,0 @@ - -######################################## -# Event Stream Interface # -######################################## - -timestamp timestamp -event varchar -user_id varchar - diff --git a/analysis/event_stream/timeseries.sql b/analysis/event_stream/timeseries.sql deleted file mode 100644 index 0a90ab3..0000000 --- a/analysis/event_stream/timeseries.sql +++ /dev/null @@ -1,8 +0,0 @@ - -SELECT - date_trunc('day', "@timestamp"), -- use second, minute, hour, day, week, month, quarter, etc - count(*) -from {{env.schema}}.snowplow_events -where "@timestamp" > getdate() - interval '1 week' - --and "@event" = 'signup' -- filter fields here -group by 1 order by 1 desc diff --git a/analysis/magento/analysis.sql b/analysis/magento/analysis.sql deleted file mode 100644 index 8226a92..0000000 --- a/analysis/magento/analysis.sql +++ /dev/null @@ -1,42 +0,0 @@ -/* - -This analysis produces a list of products that includes their: - - sku - - The Product's Name - - Quantity Ordered - - Total Revenue (Price times quantity) - - Total Cost (cost times quantity) - - The Product's cost to the store - - The Product's price to the customer - - The Profit Margin - - The Total Profit - -*/ - -with products as ( - - select * from {{env.schema}}.magento_products - -) -, order_items as ( - - select * from {{env.schema}}.magento_order_items - -) - -SELECT - products.sku, - products.name, - count(order_items.item_id) as "Quantity", - SUM(order_items.base_price) as "Total Revenue", - products.cost * count(order_items.item_id) as "Total Cost", - products.cost as "Item cost", - base_price as "price", - 1 - (products.cost / base_price) as "profit margin", - SUM(order_items.base_price) - (products.cost * count(order_items.item_id)) as "profit" -FROM order_items -RIGHT JOIN products -ON products.entity_id = order_items.product_id -WHERE order_items.base_price > 0 -GROUP BY products.sku, products.name, order_items.base_price, products.cost -ORDER BY count(order_items.item_id) desc; From 6ab97799889da7da83586efa948507db65ca187a Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Mon, 1 Aug 2016 16:46:08 -0400 Subject: [PATCH 85/88] updated to reflect the new purpose of this repo --- README.md | 42 +++++++----------------------------------- 1 file changed, 7 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 9e729b5..54fa2dd 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,11 @@ -### analyst-collective/models +### analyst-collective/analytics -A collection of data models and corresponding analysis for common data sets in SQL. These models are designed to be portable across organizations with minimal configuration. +This repository serves as an index for various dbt-based analytics packages. Please add additional packages by submitting PRs. -### Design Principles +- [Snowplow](https://github.com/fishtown-analytics/snowplow) +- [Quickbooks](https://github.com/fishtown-analytics/quickbooks) +- [Zendesk](https://github.com/analyst-collective/zendesk) +- [Mailchimp](https://github.com/analyst-collective/mailchimp) -This repository contains two primary types of objects: data models and data analyses. -##### Models -- A model is a table or view built either on top of raw data or other models. Models are not transient; they are materialized in the database. -- Models are composed of a single SQL `select` statement. Any valid SQL can be used. As such, models can provide functionality such as data cleansing, data transformation, etc. -- All models are built to be compiled and run with [dbt](https://github.com/analyst-collective/dbt). -- Models can be configured in dbt to be materialized as either views or tables. -- Model files should go into `/models` and saved with a `.sql` extension. -- Each model should be stored in its own `.sql` file. The file name will become the name of the table or view in the database. -- Other models should be referenced with the `ref` function. This function will resolve dependencies during the `compile` stage. The only tables referenced without this function should be source raw data tables. -- Models should be designed to minimize the selection from raw data tables. This minimizes the amount of mapping end users of models will need to do when configuring them for their local environment. - -##### Analysis -- Analyses are `.sql` files that can be executed within a database query tool. -- All analysis should be built on top of models, not raw data. -- All named fields in a given analysis should be named within a given model. -- Confining analysis in this way ensures portability of analysis across multiple environments. - -##### Dependencies - -- All projects can include dependencies to other projects. Dependencies - - -### Contributing -All contributions to this repository must be for analytics on top of standardized datasets. The current process for contributing is to: -- fork this repo, -- build a test dataset, -- make and test changes, and -- submit a PR. - -PRs without accompanying datasets cannot be tested and therefore will not be accepted. We suggest you use [data-generator](https://github.com/analyst-collective/data-generator) to generate your test datasets. - -We do not believe that this is the ideal workflow to facilitate the Analyst Collective vision for open source analytics. In the future, we plan to extend dbt to be a package manager. Once this is accomplished, you can own your own analytics repositories and publish them to a common index that others can use. We will update the contribution guidelines here once this is accomplished. +These packages are all installed and built using dbt. For additional information on dbt, go [here](https://github.com/analyst-collective/dbt). From 69ee6aa21bc578976ea0da56094c25b3b7ab9c10 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Mon, 1 Aug 2016 16:47:04 -0400 Subject: [PATCH 86/88] reflect deprecation --- {analysis => deprecated-analysis}/mrr/active_mrr.sql | 0 {analysis => deprecated-analysis}/mrr/total_mrr_by_month.sql | 0 {models => deprecated-models}/trello/trello_card_location.sql | 0 {models => deprecated-models}/trello/trello_model_tests.sql | 0 {models => deprecated-models}/zuora/zuora_account.sql | 0 {models => deprecated-models}/zuora/zuora_amendment.sql | 0 {models => deprecated-models}/zuora/zuora_rate_plan.sql | 0 {models => deprecated-models}/zuora/zuora_rate_plan_charge.sql | 0 {models => deprecated-models}/zuora/zuora_subscription.sql | 0 .../zuora/zuora_subscriptions_w_charges_and_amendments.sql | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename {analysis => deprecated-analysis}/mrr/active_mrr.sql (100%) rename {analysis => deprecated-analysis}/mrr/total_mrr_by_month.sql (100%) rename {models => deprecated-models}/trello/trello_card_location.sql (100%) rename {models => deprecated-models}/trello/trello_model_tests.sql (100%) rename {models => deprecated-models}/zuora/zuora_account.sql (100%) rename {models => deprecated-models}/zuora/zuora_amendment.sql (100%) rename {models => deprecated-models}/zuora/zuora_rate_plan.sql (100%) rename {models => deprecated-models}/zuora/zuora_rate_plan_charge.sql (100%) rename {models => deprecated-models}/zuora/zuora_subscription.sql (100%) rename {models => deprecated-models}/zuora/zuora_subscriptions_w_charges_and_amendments.sql (100%) diff --git a/analysis/mrr/active_mrr.sql b/deprecated-analysis/mrr/active_mrr.sql similarity index 100% rename from analysis/mrr/active_mrr.sql rename to deprecated-analysis/mrr/active_mrr.sql diff --git a/analysis/mrr/total_mrr_by_month.sql b/deprecated-analysis/mrr/total_mrr_by_month.sql similarity index 100% rename from analysis/mrr/total_mrr_by_month.sql rename to deprecated-analysis/mrr/total_mrr_by_month.sql diff --git a/models/trello/trello_card_location.sql b/deprecated-models/trello/trello_card_location.sql similarity index 100% rename from models/trello/trello_card_location.sql rename to deprecated-models/trello/trello_card_location.sql diff --git a/models/trello/trello_model_tests.sql b/deprecated-models/trello/trello_model_tests.sql similarity index 100% rename from models/trello/trello_model_tests.sql rename to deprecated-models/trello/trello_model_tests.sql diff --git a/models/zuora/zuora_account.sql b/deprecated-models/zuora/zuora_account.sql similarity index 100% rename from models/zuora/zuora_account.sql rename to deprecated-models/zuora/zuora_account.sql diff --git a/models/zuora/zuora_amendment.sql b/deprecated-models/zuora/zuora_amendment.sql similarity index 100% rename from models/zuora/zuora_amendment.sql rename to deprecated-models/zuora/zuora_amendment.sql diff --git a/models/zuora/zuora_rate_plan.sql b/deprecated-models/zuora/zuora_rate_plan.sql similarity index 100% rename from models/zuora/zuora_rate_plan.sql rename to deprecated-models/zuora/zuora_rate_plan.sql diff --git a/models/zuora/zuora_rate_plan_charge.sql b/deprecated-models/zuora/zuora_rate_plan_charge.sql similarity index 100% rename from models/zuora/zuora_rate_plan_charge.sql rename to deprecated-models/zuora/zuora_rate_plan_charge.sql diff --git a/models/zuora/zuora_subscription.sql b/deprecated-models/zuora/zuora_subscription.sql similarity index 100% rename from models/zuora/zuora_subscription.sql rename to deprecated-models/zuora/zuora_subscription.sql diff --git a/models/zuora/zuora_subscriptions_w_charges_and_amendments.sql b/deprecated-models/zuora/zuora_subscriptions_w_charges_and_amendments.sql similarity index 100% rename from models/zuora/zuora_subscriptions_w_charges_and_amendments.sql rename to deprecated-models/zuora/zuora_subscriptions_w_charges_and_amendments.sql From b72a3bc88bf4a1f52614b7162f2a26b0649e6a14 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Mon, 1 Aug 2016 16:47:28 -0400 Subject: [PATCH 87/88] . --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 54fa2dd..667f4d9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -### analyst-collective/analytics +# analyst-collective/analytics This repository serves as an index for various dbt-based analytics packages. Please add additional packages by submitting PRs. From a5a8c8e66f51f1c684aa8587fb84d110886aa903 Mon Sep 17 00:00:00 2001 From: Tristan Handy Date: Fri, 23 Sep 2016 15:10:50 -0400 Subject: [PATCH 88/88] update readme to include new stripe repo --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 667f4d9..9bd6c0a 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ This repository serves as an index for various dbt-based analytics packages. Please add additional packages by submitting PRs. +- [Stripe](https://github.com/fishtown-analytics/stripe) - [Snowplow](https://github.com/fishtown-analytics/snowplow) - [Quickbooks](https://github.com/fishtown-analytics/quickbooks) - [Zendesk](https://github.com/analyst-collective/zendesk)