From cbe22f8b2bed9802f1f2a7d103693cdb16f336b4 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 14 Jun 2021 16:10:44 +0200 Subject: [PATCH 1/3] Add admins_activity_logs to tap --- .../schemas/admins_activity_logs.json | 41 +++++++++++++++++++ tap_intercom/streams.py | 9 ++++ 2 files changed, 50 insertions(+) create mode 100644 tap_intercom/schemas/admins_activity_logs.json diff --git a/tap_intercom/schemas/admins_activity_logs.json b/tap_intercom/schemas/admins_activity_logs.json new file mode 100644 index 0000000..3c3af4f --- /dev/null +++ b/tap_intercom/schemas/admins_activity_logs.json @@ -0,0 +1,41 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": ["null", "string"] + }, + "activity_type": { + "type": ["null", "string"] + }, + "activity_description": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "metadata": { + "type": ["null", "object"], + "additionalProperties": true + }, + "performed_by": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "type" : { + "type": ["null", "string"] + }, + "id" : { + "type": ["null", "string"] + }, + "ip" : { + "type": ["null", "string"] + }, + "email" : { + "type": ["null", "string"] + } + } + } + } +} \ No newline at end of file diff --git a/tap_intercom/streams.py b/tap_intercom/streams.py index 7bbdb35..445218d 100644 --- a/tap_intercom/streams.py +++ b/tap_intercom/streams.py @@ -155,6 +155,15 @@ def build_query(body, value, starting_after=None): 'key_properties': ['id'], 'replication_method': 'FULL_TABLE' }, + 'admins_activity_logs': { + 'key_properties': ['id'], + 'replication_method': 'INCREMENTAL', + 'replication_keys': ['created_at'], + 'bookmark_type': 'datetime', + 'data_key': 'activity_logs', + 'path': 'admins/activity_logs', + 'bookmark_query_field': 'created_at_after' + }, } From 6d3a9d76b34f8fd81329b9af2a13c45e1285bfce Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 14 Jun 2021 16:33:24 +0200 Subject: [PATCH 2/3] Update README with new activity logs stream --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index ec85189..e83f32a 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ This tap: - Pulls raw data from the [Intercom v1.4 API](https://developers.intercom.com/intercom-api-reference/reference#introduction) - Extracts the following resources: - [Admins](https://developers.intercom.com/intercom-api-reference/reference#list-admins) + - [Admins Activity Logs](https://developers.intercom.com/intercom-api-reference/reference#view-admin-activity-logs) - [Companies](https://developers.intercom.com/intercom-api-reference/reference#list-companies) - [Conversations](https://developers.intercom.com/intercom-api-reference/reference#list-conversations) - [Conversation Parts](https://developers.intercom.com/intercom-api-reference/reference#get-a-single-conversation) @@ -34,6 +35,15 @@ This tap: - Replication strategy: FULL_TABLE - Transformations: none +[admins_activity_logs](https://developers.intercom.com/intercom-api-reference/reference#view-admin-activity-logs) +- Endpoint: https://api.intercom.io/admins/activity_logs +- Primary key fields: id +- Foreign key fields: performed_by > id +- Replication strategy: INCREMENTAL + - Bookmark: created_at (date-time) + - Bookmark query field: created_at_after +- Transformations: none + [companies](https://developers.intercom.com/intercom-api-reference/reference#list-companies) - Endpoint: https://api.intercom.io/companies - Primary key fields: id From bbbde7ec775e6f090541d6988f48586028caded2 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 15 Jun 2021 17:43:38 +0200 Subject: [PATCH 3/3] Add support for epoch bookmark field --- state.json.example | 11 ----------- tap_intercom/streams.py | 4 +++- tap_intercom/sync.py | 8 +++++++- 3 files changed, 10 insertions(+), 13 deletions(-) delete mode 100644 state.json.example diff --git a/state.json.example b/state.json.example deleted file mode 100644 index a4fb638..0000000 --- a/state.json.example +++ /dev/null @@ -1,11 +0,0 @@ -{ - "currently_syncing": "teams", - "bookmarks": { - "users": "2019-09-27T22:34:39.000000Z", - "companies": "2019-09-28T15:30:26.000000Z", - "conversations": "2019-09-28T18:23:53Z", - "segments": "2019-09-27T22:40:30.000000Z", - "company_segments": "2019-09-27T22:41:24.000000Z", - "leads": "2019-09-27T22:16:30.000000Z" - } -} \ No newline at end of file diff --git a/tap_intercom/streams.py b/tap_intercom/streams.py index 445218d..6ace13a 100644 --- a/tap_intercom/streams.py +++ b/tap_intercom/streams.py @@ -15,6 +15,7 @@ # scroll_type: always, historical, or never; default never # interpolate_page: True, False (to determine start page based on total pages and bookmark) # default: False +# bookmark_query_field_to_epoch: True, False (convert datetime bookmark to epoch upon query) def build_query(body, value, starting_after=None): # Update both the > and the = queries with our bookmark @@ -162,7 +163,8 @@ def build_query(body, value, starting_after=None): 'bookmark_type': 'datetime', 'data_key': 'activity_logs', 'path': 'admins/activity_logs', - 'bookmark_query_field': 'created_at_after' + 'bookmark_query_field': 'created_at_after', + 'bookmark_query_field_to_epoch': True }, } diff --git a/tap_intercom/sync.py b/tap_intercom/sync.py index ddc2043..43c8d30 100644 --- a/tap_intercom/sync.py +++ b/tap_intercom/sync.py @@ -97,6 +97,7 @@ def process_records(catalog, #pylint: disable=too-many-branches LOGGER.error('Transformer Error: {}'.format(err)) LOGGER.error('Stream: {}, record: {}'.format(stream_name, record)) raise + # Reset max_bookmark_value to new value if higher if transformed_record.get(bookmark_field): if max_bookmark_value is None or \ @@ -284,6 +285,8 @@ def sync_endpoint(client, #pylint: disable=too-many-branches search_query = endpoint_config.get('search_query') request_body = build_query(search_query, max_bookmark_int) + bookmark_query_field_to_epoch = endpoint_config.get('bookmark_query_field_to_epoch', False) + i = 1 while next_url is not None: # Need URL querystring for 1st page; subsequent pages provided by next_url @@ -291,7 +294,10 @@ def sync_endpoint(client, #pylint: disable=too-many-branches if i == 1 and not is_scrolling: if bookmark_query_field: if bookmark_type == 'datetime': - params[bookmark_query_field] = updated_since_days + if bookmark_query_field_to_epoch: + params[bookmark_query_field] = max_bookmark_int + else: + params[bookmark_query_field] = updated_since_days elif bookmark_type == 'integer': params[bookmark_query_field] = last_integer if params != {}: