From 914d48ae89995177aacd11cc3e3037a6cc22334b Mon Sep 17 00:00:00 2001 From: randal boyle Date: Wed, 20 Sep 2023 17:16:04 +0100 Subject: [PATCH 1/3] fixing error that is thrown when variant column holds a non dict (list or string), also changing query to pull row where all variant columns have a value --- dbt_coves/tasks/generate/sources.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/dbt_coves/tasks/generate/sources.py b/dbt_coves/tasks/generate/sources.py index 6915d799..b680fdfd 100644 --- a/dbt_coves/tasks/generate/sources.py +++ b/dbt_coves/tasks/generate/sources.py @@ -36,13 +36,13 @@ def register_parser(cls, sub_parsers, base_subparser): "--schemas", type=str, help="Comma separated list of schemas where raw data resides, " - "i.e. 'RAW_SALESFORCE,RAW_HUBSPOT'", + "i.e. 'RAW_SALESFORCE,RAW_HUBSPOT'", ) subparser.add_argument( "--select-relations", type=str, help="Comma separated list of relations where raw data resides, " - "i.e. 'RAW_HUBSPOT_PRODUCTS,RAW_SALESFORCE_USERS'", + "i.e. 'RAW_HUBSPOT_PRODUCTS,RAW_SALESFORCE_USERS'", ) subparser.add_argument( "--exclude-relations", @@ -53,31 +53,31 @@ def register_parser(cls, sub_parsers, base_subparser): "--sources-destination", type=str, help="Where sources yml files will be generated, default: " - "'models/staging/{{schema}}/sources.yml'", + "'models/staging/{{schema}}/sources.yml'", ) subparser.add_argument( "--models-destination", type=str, help="Where models sql files will be generated, default: " - "'models/staging/{{schema}}/{{relation}}.sql'", + "'models/staging/{{schema}}/{{relation}}.sql'", ) subparser.add_argument( "--model-props-destination", type=str, help="Where models yml files will be generated, default: " - "'models/staging/{{schema}}/{{relation}}.yml'", + "'models/staging/{{schema}}/{{relation}}.yml'", ) subparser.add_argument( "--update-strategy", type=str, help="Action to perform when a property file already exists: " - "'update', 'recreate', 'fail', 'ask' (per file)", + "'update', 'recreate', 'fail', 'ask' (per file)", ) subparser.add_argument( "--templates-folder", type=str, help="Folder with jinja templates that override default " - "sources generation templates, i.e. 'templates'", + "sources generation templates, i.e. 'templates'", ) subparser.add_argument( "--metadata", @@ -317,7 +317,7 @@ def get_nested_keys(self, json_cols, relation): config_db = "" _, data = self.adapter.execute( f"SELECT {', '.join(json_cols)} FROM {config_db}{relation.schema}.{relation.name} \ - limit 1", + WHERE {' IS NOT NULL AND '.join([' CAST(' + sub + ' AS VARCHAR) ' for sub in json_cols])} IS NOT NULL limit 1", fetch=True, ) result = dict() @@ -325,11 +325,13 @@ def get_nested_keys(self, json_cols, relation): for idx, json_col in enumerate(json_cols): value = data.columns[idx] try: - nested_key_names = list(json.loads(value[0]).keys()) - result[json_col] = {} - for key_name in nested_key_names: - result[json_col][key_name] = self.get_default_metadata_item(key_name) - self.add_metadata_to_nested(relation, result, json_col) + nested_object = json.loads(value[0]) + if type(nested_object) is dict: + nested_key_names = list(nested_object.keys()) + result[json_col] = {} + for key_name in nested_key_names: + result[json_col][key_name] = self.get_default_metadata_item(key_name) + self.add_metadata_to_nested(relation, result, json_col) except TypeError: console.print( f"Column {json_col} in relation {relation.name} contains invalid JSON.\n" From c9b1596bbe6746ff80f27df45962e9b29665cc30 Mon Sep 17 00:00:00 2001 From: randal boyle Date: Wed, 20 Sep 2023 17:20:54 +0100 Subject: [PATCH 2/3] fixing error that is thrown when variant column holds a non dict (list or string), also changing query to pull row where all variant columns have a value --- dbt_coves/tasks/generate/sources.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dbt_coves/tasks/generate/sources.py b/dbt_coves/tasks/generate/sources.py index b680fdfd..5d8e0dc1 100644 --- a/dbt_coves/tasks/generate/sources.py +++ b/dbt_coves/tasks/generate/sources.py @@ -36,13 +36,13 @@ def register_parser(cls, sub_parsers, base_subparser): "--schemas", type=str, help="Comma separated list of schemas where raw data resides, " - "i.e. 'RAW_SALESFORCE,RAW_HUBSPOT'", + "i.e. 'RAW_SALESFORCE,RAW_HUBSPOT'", ) subparser.add_argument( "--select-relations", type=str, help="Comma separated list of relations where raw data resides, " - "i.e. 'RAW_HUBSPOT_PRODUCTS,RAW_SALESFORCE_USERS'", + "i.e. 'RAW_HUBSPOT_PRODUCTS,RAW_SALESFORCE_USERS'", ) subparser.add_argument( "--exclude-relations", @@ -53,31 +53,31 @@ def register_parser(cls, sub_parsers, base_subparser): "--sources-destination", type=str, help="Where sources yml files will be generated, default: " - "'models/staging/{{schema}}/sources.yml'", + "'models/staging/{{schema}}/sources.yml'", ) subparser.add_argument( "--models-destination", type=str, help="Where models sql files will be generated, default: " - "'models/staging/{{schema}}/{{relation}}.sql'", + "'models/staging/{{schema}}/{{relation}}.sql'", ) subparser.add_argument( "--model-props-destination", type=str, help="Where models yml files will be generated, default: " - "'models/staging/{{schema}}/{{relation}}.yml'", + "'models/staging/{{schema}}/{{relation}}.yml'", ) subparser.add_argument( "--update-strategy", type=str, help="Action to perform when a property file already exists: " - "'update', 'recreate', 'fail', 'ask' (per file)", + "'update', 'recreate', 'fail', 'ask' (per file)", ) subparser.add_argument( "--templates-folder", type=str, help="Folder with jinja templates that override default " - "sources generation templates, i.e. 'templates'", + "sources generation templates, i.e. 'templates'", ) subparser.add_argument( "--metadata", From c716febdca5692c918c5b3d93d97f379d310f7d0 Mon Sep 17 00:00:00 2001 From: randal boyle Date: Wed, 20 Sep 2023 17:23:30 +0100 Subject: [PATCH 3/3] fixing error that is thrown when variant column holds a non dict (list or string), also changing query to pull row where all variant columns have a value --- dbt_coves/tasks/generate/sources.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt_coves/tasks/generate/sources.py b/dbt_coves/tasks/generate/sources.py index 5d8e0dc1..221fb7fc 100644 --- a/dbt_coves/tasks/generate/sources.py +++ b/dbt_coves/tasks/generate/sources.py @@ -317,7 +317,8 @@ def get_nested_keys(self, json_cols, relation): config_db = "" _, data = self.adapter.execute( f"SELECT {', '.join(json_cols)} FROM {config_db}{relation.schema}.{relation.name} \ - WHERE {' IS NOT NULL AND '.join([' CAST(' + sub + ' AS VARCHAR) ' for sub in json_cols])} IS NOT NULL limit 1", + WHERE {' IS NOT NULL AND '.join([' CAST(' + sub + ' AS VARCHAR) ' for sub in json_cols])} \ + IS NOT NULL limit 1", fetch=True, ) result = dict()