Feature/url tags super

.buildkite/pipeline.yml

@@ -3,7 +3,7 @@ steps:
     key: "run-dbt-postgres"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
@@ -18,7 +18,7 @@ steps:
     key: "run_dbt_snowflake"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
@@ -35,7 +35,7 @@ steps:
     key: "run_dbt_bigquery"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
@@ -47,7 +47,7 @@ steps:
     key: "run_dbt_redshift"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
@@ -62,7 +62,7 @@ steps:
     key: "run_dbt_databricks"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"

.buildkite/scripts/run_models.sh

@@ -20,6 +20,10 @@ dbt seed --target "$db" --full-refresh
 dbt source freshness --target "$db" || echo "...Only verifying freshness runs…"
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
+if [ "$db" = "bigquery" ] || [ "$db" = "redshift" ] || [ "$db" = "postgres" ] || [ "$db" = "snowflake" ]; then
+dbt run --vars '{facebook_ads_creative_history_identifier: facebook_ads_creative_history_json_data}' --target "$db" --full-refresh
+dbt test --target "$db"
+fi
 dbt run --vars '{ad_reporting__url_report__using_null_filter: false, facebook_ads__using_demographics_country: true, facebook_ads__using_demographics_region: true}' --target "$db" --full-refresh
 dbt test --vars '{ad_reporting__url_report__using_null_filter: false, facebook_ads__using_demographics_country: true, facebook_ads__using_demographics_region: true}' --target "$db"
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"

CHANGELOG.md

@@ -1,3 +1,17 @@
+# dbt_facebook_ads v1.4.1
+[PR #65](https://github.com/fivetran/dbt_facebook_ads/pull/65) includes the following updates:
+
+## Bug Fix
+- Updates URL tag processing to handle native JSON datatypes across all supported warehouses, preventing errors when `url_tags` columns are stored as JSON/JSONB/VARIANT/SUPER instead of strings. 
+
+## Feature Update
+- Adds the `get_column_datatype()` macro to retrieve the datatype of a specific column. 
+- Updates the `get_url_tags_query()` macro with datatype detection to handle both native JSON columns and JSON-like string columns across BigQuery (JSON), Snowflake (VARIANT), Redshift (SUPER), and PostgreSQL (JSONB).
+
+## Under the Hood
+- Adds native JSON testing to `integration_tests`.
+- Updates the `vertical_sum_conversion_value` integrity test to validate both `conversions_value` and `conversions`.
+
 # dbt_facebook_ads v1.4.0
 
 [PR #63](https://github.com/fivetran/dbt_facebook_ads/pull/63) includes the following updates:

README.md

@@ -1,21 +1,6 @@
 <!--section="facebook-ads_transformation_model"-->
 # Facebook Ads dbt Package
 
-<p align="left">
-    <a alt="License"
-        href="https://github.com/fivetran/dbt_facebook_ads/blob/main/LICENSE">
-        <img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" /></a>
-    <a alt="dbt-core">
-        <img src="https://img.shields.io/badge/dbt_Core™_version->=1.3.0,_<3.0.0-orange.svg" /></a>
-    <a alt="Maintained?">
-        <img src="https://img.shields.io/badge/Maintained%3F-yes-green.svg" /></a>
-    <a alt="PRs">
-        <img src="https://img.shields.io/badge/Contributions-welcome-blueviolet" /></a>
-    <a alt="Fivetran Quickstart Compatible"
-        href="https://fivetran.com/docs/transformations/data-models/quickstart-management#quickstartmanagement">
-        <img src="https://img.shields.io/badge/Fivetran_Quickstart_Compatible%3F-yes-green.svg" /></a>
-</p>
-
 This dbt package transforms data from Fivetran's Facebook Ads connector into analytics-ready tables.
 
 ## Resources
@@ -29,6 +14,8 @@ This dbt package transforms data from Fivetran's Facebook Ads connector into ana
   - [dbt Docs](https://fivetran.github.io/dbt_facebook_ads/#!/overview)
   - [DAG](https://fivetran.github.io/dbt_facebook_ads/#!/overview?g_v=1)
   - [Changelog](https://github.com/fivetran/dbt_facebook_ads/blob/main/CHANGELOG.md)
+- dbt Core™ supported versions
+  - `>=1.3.0, <3.0.0`
 
 ## What does this dbt package do?
 This package enables you to better understand the performance of your ads across varying grains and produces modeled tables that leverage Facebook Ads data. It creates enriched models with metrics focused on account, campaign, ad set, ad, and geographic reporting.

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'facebook_ads'
-version: '1.4.0'
+version: '1.4.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<3.0.0"]
 models:

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'facebook_ads_integration_tests'
-version: '1.4.0'
+version: '1.4.1'
 profile: 'integration_tests'
 config-version: 2
 
@@ -40,34 +40,97 @@ vars:
       where_sql: source_relation != 'no_purchase_source' # @docs-ignore
 
 models:
+  +persist_docs:
+    relation: true
+    columns: "{{ false if target.type in ('databricks') else true }}"
   +schema: "facebook_ads_{{ var('directed_schema','dev') }}"
 
 seeds:
   +docs:
     show: false
   facebook_ads_integration_tests:
-    +column_types:
-      id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
-      account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
-      campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
-      ad_set_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
-      ad_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
-      creative_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
-      page_link: "{{ 'string' if target.type in ['bigquery','spark','databricks'] else 'varchar' }}"
-      template_page_link: "{{ 'string' if target.type in ['bigquery','spark','databricks'] else 'varchar' }}"
-      _fivetran_synced: "timestamp"
-      updated_time: "timestamp"
+    facebook_ads_account_history_data:
+      +column_types:
+        id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        _fivetran_synced: "timestamp"
+    facebook_ads_ad_history_data:
+      +column_types:
+        id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        ad_set_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        creative_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        _fivetran_synced: "timestamp"
+        updated_time: "timestamp"
+    facebook_ads_ad_set_history_data:
+      +column_types:
+        id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        _fivetran_synced: "timestamp"
+        updated_time: "timestamp"
+    facebook_ads_campaign_history_data:
+      +column_types:
+        id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        _fivetran_synced: "timestamp"
+        updated_time: "timestamp"
+    facebook_ads_creative_history_data:
+      +column_types:
+        id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        page_link: "{{ 'string' if target.type in ['bigquery','spark','databricks'] else 'varchar' }}"
+        template_page_link: "{{ 'string' if target.type in ['bigquery','spark','databricks'] else 'varchar' }}"
+        _fivetran_synced: "timestamp"
+        url_tags: "{{ 'string' if target.type in ['bigquery','spark','databricks'] else 'varchar' }}"
+    facebook_ads_creative_history_json_data:
+      +enabled: "{{ target.type in ('bigquery', 'redshift', 'postgres', 'snowflake') }}" # @docs-ignore
+      # +enabled: false # @docs-include
+      +column_types:
+        id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        page_link: "{{ 'string' if target.type in ['bigquery','spark','databricks'] else 'varchar' }}"
+        template_page_link: "{{ 'string' if target.type in ['bigquery','spark','databricks'] else 'varchar' }}"
+        _fivetran_synced: "timestamp"
+        url_tags_string: "{{ 'string' if target.type in ['bigquery','spark','databricks'] else 'varchar' }}"
+      # Load as VARCHAR first, then convert to native JSON type in a post-hook.
+      # This is necessary for testing native JSON datatypes with seed data.
+      +post-hook:
+        - "alter table {{ this }} add column url_tags {{ 'super' if target.type == 'redshift' else 'variant' if target.type == 'snowflake' else 'json' }} {{ 'default null' if target.type != 'bigquery' }}"
+        - "update {{ this }} set url_tags = {{ 'json_parse(url_tags_string)' if target.type == 'redshift' else 'url_tags_string::jsonb' if target.type == 'postgres' else 'parse_json(url_tags_string)' }} {{ 'where url_tags_string is not null' if target.type == 'bigquery' }}"
+    facebook_ads_basic_ad_data:
+      +column_types:
+        ad_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
     facebook_ads_basic_ad_actions_data:
       +column_types:
+        ad_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        _fivetran_synced: "timestamp"
         value: "float"
         inline: "float"
         _1_d_view: "float"
         _7_d_click: "float"
+    facebook_ads_basic_ad_action_values_data:
+      +column_types:
+        ad_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        _fivetran_synced: "timestamp"
+    facebook_ads_demographics_country_data:
+      +column_types:
+        account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        _fivetran_synced: "timestamp"
     facebook_ads_demographics_country_actions_data:
       +column_types:
+        account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        _fivetran_synced: "timestamp"
         value: "float"
+    facebook_ads_demographics_region_data:
+      +column_types:
+        account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        _fivetran_synced: "timestamp"
     facebook_ads_demographics_region_actions_data:
       +column_types:
+        account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        _fivetran_synced: "timestamp"
         value: "float"
 
 dispatch: