From fbfa7d5bd3ae42474cf18beaf69402f5060606e7 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 14:09:01 +0000 Subject: [PATCH 01/15] Re-enable ClickHouse in CLI tests Co-Authored-By: Itamar Hartstein --- .github/workflows/test-all-warehouses.yml | 2 +- .github/workflows/test-warehouse.yml | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-all-warehouses.yml b/.github/workflows/test-all-warehouses.yml index 6a969c825..0960fc98e 100644 --- a/.github/workflows/test-all-warehouses.yml +++ b/.github/workflows/test-all-warehouses.yml @@ -33,7 +33,7 @@ jobs: matrix: dbt-version: ${{ inputs.dbt-version && fromJSON(format('["{0}"]', inputs.dbt-version)) || fromJSON('[null]') }} warehouse-type: - [postgres, snowflake, bigquery, redshift, databricks_catalog, athena] + [postgres, snowflake, bigquery, redshift, databricks_catalog, athena, clickhouse] uses: ./.github/workflows/test-warehouse.yml with: warehouse-type: ${{ matrix.warehouse-type }} diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index fa5195305..836b26c64 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -14,6 +14,7 @@ on: - databricks_catalog - spark - athena + - clickhouse elementary-ref: type: string required: false @@ -103,10 +104,10 @@ jobs: working-directory: ${{ env.E2E_DBT_PROJECT_DIR }} run: docker compose up -d postgres - # - name: Start Clickhouse - # if: inputs.warehouse-type == 'clickhouse' - # working-directory: ${{ env.E2E_DBT_PROJECT_DIR }} - # run: docker compose up -d clickhouse + - name: Start Clickhouse + if: inputs.warehouse-type == 'clickhouse' + working-directory: ${{ env.E2E_DBT_PROJECT_DIR }} + run: docker compose up -d clickhouse - name: Setup Python uses: actions/setup-python@v4 From dee008c7b22cca9106c8dac0d10858144864c0b4 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 14:12:24 +0000 Subject: [PATCH 02/15] Fix prettier formatting for warehouse-type array Co-Authored-By: Itamar Hartstein --- .github/workflows/test-all-warehouses.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-all-warehouses.yml b/.github/workflows/test-all-warehouses.yml index 0960fc98e..858207ae2 100644 --- a/.github/workflows/test-all-warehouses.yml +++ b/.github/workflows/test-all-warehouses.yml @@ -33,7 +33,15 @@ jobs: matrix: dbt-version: ${{ inputs.dbt-version && fromJSON(format('["{0}"]', inputs.dbt-version)) || fromJSON('[null]') }} warehouse-type: - [postgres, snowflake, bigquery, redshift, databricks_catalog, athena, clickhouse] + [ + postgres, + snowflake, + bigquery, + redshift, + databricks_catalog, + athena, + clickhouse, + ] uses: ./.github/workflows/test-warehouse.yml with: warehouse-type: ${{ matrix.warehouse-type }} From 35c94c1d0d27525be918027f2d11ad2a53b4e4f3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 15:14:25 +0000 Subject: [PATCH 03/15] Add clickhouse to Seed e2e dbt project step Co-Authored-By: Itamar Hartstein --- .github/workflows/test-warehouse.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index 836b26c64..17bc7fd18 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -162,7 +162,7 @@ jobs: - name: Seed e2e dbt project working-directory: ${{ env.E2E_DBT_PROJECT_DIR }} - if: inputs.warehouse-type == 'postgres' || inputs.generate-data + if: inputs.warehouse-type == 'postgres' || inputs.warehouse-type == 'clickhouse' || inputs.generate-data run: | python generate_data.py dbt seed -f --target "${{ inputs.warehouse-type }}" From 1fc1952c185153e44935ea8df5a83047b6a5df54 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 15:40:40 +0000 Subject: [PATCH 04/15] Fix get_elementary_database_and_schema to use actual elementary schema Co-Authored-By: Itamar Hartstein --- .../macros/get_elementary_database_and_schema.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/elementary/monitor/dbt_project/macros/get_elementary_database_and_schema.sql b/elementary/monitor/dbt_project/macros/get_elementary_database_and_schema.sql index 16ad5f4d9..6ee06911e 100644 --- a/elementary/monitor/dbt_project/macros/get_elementary_database_and_schema.sql +++ b/elementary/monitor/dbt_project/macros/get_elementary_database_and_schema.sql @@ -1,5 +1,9 @@ {% macro get_elementary_database_and_schema() %} - {% set database, schema = elementary.target_database(), target.schema %} + {# Use ref() to get the actual schema where elementary tables are created, + accounting for any custom schema configuration in the user's dbt_project.yml #} + {% set elementary_relation = ref('elementary', 'dbt_models') %} + {% set database = elementary_relation.database %} + {% set schema = elementary_relation.schema %} {% if database %} {% do return(database ~ '.' ~ schema) %} {% else %} From c92b6a9c3a71e4ef3304a502d2c79767f3061e4e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 15:46:08 +0000 Subject: [PATCH 05/15] Revert "Fix get_elementary_database_and_schema to use actual elementary schema" This reverts commit 1fc1952c185153e44935ea8df5a83047b6a5df54. --- .../macros/get_elementary_database_and_schema.sql | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/elementary/monitor/dbt_project/macros/get_elementary_database_and_schema.sql b/elementary/monitor/dbt_project/macros/get_elementary_database_and_schema.sql index 6ee06911e..16ad5f4d9 100644 --- a/elementary/monitor/dbt_project/macros/get_elementary_database_and_schema.sql +++ b/elementary/monitor/dbt_project/macros/get_elementary_database_and_schema.sql @@ -1,9 +1,5 @@ {% macro get_elementary_database_and_schema() %} - {# Use ref() to get the actual schema where elementary tables are created, - accounting for any custom schema configuration in the user's dbt_project.yml #} - {% set elementary_relation = ref('elementary', 'dbt_models') %} - {% set database = elementary_relation.database %} - {% set schema = elementary_relation.schema %} + {% set database, schema = elementary.target_database(), target.schema %} {% if database %} {% do return(database ~ '.' ~ schema) %} {% else %} From a3bef9cf6897af76b604ac79f74e9c308b2ad6a3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:13:27 +0000 Subject: [PATCH 06/15] Fix ClickHouse column alias handling in populate_test_alerts macro Co-Authored-By: Itamar Hartstein --- .../macros/alerts/population/test_alerts.sql | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql index 8ebe9eb0e..31089372d 100644 --- a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql @@ -5,16 +5,23 @@ {% set raw_test_alerts_agate = run_query(elementary_cli.populate_test_alerts_query(days_back)) %} {% set raw_test_alerts = elementary.agate_to_dicts(raw_test_alerts_agate) %} {% for raw_test_alert in raw_test_alerts %} - {% set test_type = raw_test_alert.alert_type %} + {# ClickHouse may return original column names instead of aliases, so we handle both cases #} + {% set test_type = raw_test_alert.get('alert_type', raw_test_alert.get('test_type')) %} {% set status = raw_test_alert.status | lower %} + {# ClickHouse may return original column names instead of aliases, so we handle both cases #} + {% set alert_id = raw_test_alert.get('alert_id', raw_test_alert.get('id')) %} + {% set test_rows_sample = none %} {%- if not disable_samples and ((test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status != 'error')) -%} - {% set test_rows_sample = elementary_cli.get_test_rows_sample(raw_test_alert.result_rows, test_result_rows_agate.get(raw_test_alert.alert_id)) %} + {% set test_rows_sample = elementary_cli.get_test_rows_sample(raw_test_alert.result_rows, test_result_rows_agate.get(alert_id)) %} {%- endif -%} + {% set sub_type = raw_test_alert.get('sub_type', raw_test_alert.get('test_sub_type')) %} + {% set alert_description = raw_test_alert.get('alert_description', raw_test_alert.get('test_results_description')) %} + {% set alert_results_query = raw_test_alert.get('alert_results_query', raw_test_alert.get('test_results_query')) %} {% set test_alert_data = { - 'id': raw_test_alert.alert_id, + 'id': alert_id, 'alert_class_id': raw_test_alert.alert_class_id, 'model_unique_id': raw_test_alert.model_unique_id, 'test_unique_id': raw_test_alert.test_unique_id, @@ -24,12 +31,12 @@ 'table_name': raw_test_alert.table_name, 'column_name': raw_test_alert.column_name, 'test_type': test_type, - 'test_sub_type': raw_test_alert.sub_type, + 'test_sub_type': sub_type, 'test_description': raw_test_alert.test_description, - 'test_results_description': raw_test_alert.alert_description, + 'test_results_description': alert_description, 'owners': raw_test_alert.owners, 'tags': raw_test_alert.tags, - 'test_results_query': raw_test_alert.alert_results_query, + 'test_results_query': alert_results_query, 'test_rows_sample': test_rows_sample, 'other': raw_test_alert.other, 'test_name': raw_test_alert.test_name, @@ -46,11 +53,11 @@ 'job_url': raw_test_alert.job_url, 'job_run_url': raw_test_alert.job_run_url, 'orchestrator': raw_test_alert.orchestrator - } + } %} {% set test_alert = elementary_cli.generate_alert_object( - raw_test_alert.alert_id, + alert_id, raw_test_alert.alert_class_id, 'test', raw_test_alert.detected_at, From 5a7efb8317bda1667397f3dbeb50123facfe9097 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:23:27 +0000 Subject: [PATCH 07/15] Use .get() for all field accesses in populate_test_alerts to handle ClickHouse column names Co-Authored-By: Itamar Hartstein --- .../macros/alerts/population/test_alerts.sql | 76 ++++++++++--------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql index 31089372d..5e1b05cc5 100644 --- a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql @@ -5,63 +5,65 @@ {% set raw_test_alerts_agate = run_query(elementary_cli.populate_test_alerts_query(days_back)) %} {% set raw_test_alerts = elementary.agate_to_dicts(raw_test_alerts_agate) %} {% for raw_test_alert in raw_test_alerts %} - {# ClickHouse may return original column names instead of aliases, so we handle both cases #} - {% set test_type = raw_test_alert.get('alert_type', raw_test_alert.get('test_type')) %} - {% set status = raw_test_alert.status | lower %} - - {# ClickHouse may return original column names instead of aliases, so we handle both cases #} + {# ClickHouse may return original column names instead of aliases, so we use .get() for all field accesses #} {% set alert_id = raw_test_alert.get('alert_id', raw_test_alert.get('id')) %} + {% set test_type = raw_test_alert.get('alert_type', raw_test_alert.get('test_type')) %} + {% set status = raw_test_alert.get('status', '') | lower %} + {% set sub_type = raw_test_alert.get('sub_type', raw_test_alert.get('test_sub_type')) %} + {% set alert_description = raw_test_alert.get('alert_description', raw_test_alert.get('test_results_description')) %} + {% set alert_results_query = raw_test_alert.get('alert_results_query', raw_test_alert.get('test_results_query')) %} + {% set test_description = raw_test_alert.get('test_description', raw_test_alert.get('description')) %} + {% set test_meta = raw_test_alert.get('test_meta', raw_test_alert.get('meta')) %} + {% set model_meta = raw_test_alert.get('model_meta') %} + {% set result_rows = raw_test_alert.get('result_rows') %} {% set test_rows_sample = none %} {%- if not disable_samples and ((test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status != 'error')) -%} - {% set test_rows_sample = elementary_cli.get_test_rows_sample(raw_test_alert.result_rows, test_result_rows_agate.get(alert_id)) %} + {% set test_rows_sample = elementary_cli.get_test_rows_sample(result_rows, test_result_rows_agate.get(alert_id)) %} {%- endif -%} - {% set sub_type = raw_test_alert.get('sub_type', raw_test_alert.get('test_sub_type')) %} - {% set alert_description = raw_test_alert.get('alert_description', raw_test_alert.get('test_results_description')) %} - {% set alert_results_query = raw_test_alert.get('alert_results_query', raw_test_alert.get('test_results_query')) %} {% set test_alert_data = { 'id': alert_id, - 'alert_class_id': raw_test_alert.alert_class_id, - 'model_unique_id': raw_test_alert.model_unique_id, - 'test_unique_id': raw_test_alert.test_unique_id, - 'detected_at': raw_test_alert.detected_at, - 'database_name': raw_test_alert.database_name, - 'schema_name': raw_test_alert.schema_name, - 'table_name': raw_test_alert.table_name, - 'column_name': raw_test_alert.column_name, + 'alert_class_id': raw_test_alert.get('alert_class_id'), + 'model_unique_id': raw_test_alert.get('model_unique_id'), + 'test_unique_id': raw_test_alert.get('test_unique_id'), + 'detected_at': raw_test_alert.get('detected_at'), + 'database_name': raw_test_alert.get('database_name'), + 'schema_name': raw_test_alert.get('schema_name'), + 'table_name': raw_test_alert.get('table_name'), + 'column_name': raw_test_alert.get('column_name'), 'test_type': test_type, 'test_sub_type': sub_type, - 'test_description': raw_test_alert.test_description, + 'test_description': test_description, 'test_results_description': alert_description, - 'owners': raw_test_alert.owners, - 'tags': raw_test_alert.tags, + 'owners': raw_test_alert.get('owners'), + 'tags': raw_test_alert.get('tags'), 'test_results_query': alert_results_query, 'test_rows_sample': test_rows_sample, - 'other': raw_test_alert.other, - 'test_name': raw_test_alert.test_name, - 'test_short_name': raw_test_alert.test_short_name, - 'test_params': raw_test_alert.test_params, - 'severity': raw_test_alert.severity, - 'test_meta': raw_test_alert.test_meta, - 'model_meta': raw_test_alert.model_meta, + 'other': raw_test_alert.get('other'), + 'test_name': raw_test_alert.get('test_name'), + 'test_short_name': raw_test_alert.get('test_short_name'), + 'test_params': raw_test_alert.get('test_params'), + 'severity': raw_test_alert.get('severity'), + 'test_meta': test_meta, + 'model_meta': model_meta, 'status': status, - 'elementary_unique_id': raw_test_alert.elementary_unique_id, - 'job_id': raw_test_alert.job_id, - 'job_name': raw_test_alert.job_name, - 'job_run_id': raw_test_alert.job_run_id, - 'job_url': raw_test_alert.job_url, - 'job_run_url': raw_test_alert.job_run_url, - 'orchestrator': raw_test_alert.orchestrator + 'elementary_unique_id': raw_test_alert.get('elementary_unique_id'), + 'job_id': raw_test_alert.get('job_id'), + 'job_name': raw_test_alert.get('job_name'), + 'job_run_id': raw_test_alert.get('job_run_id'), + 'job_url': raw_test_alert.get('job_url'), + 'job_run_url': raw_test_alert.get('job_run_url'), + 'orchestrator': raw_test_alert.get('orchestrator') } %} {% set test_alert = elementary_cli.generate_alert_object( alert_id, - raw_test_alert.alert_class_id, + raw_test_alert.get('alert_class_id'), 'test', - raw_test_alert.detected_at, - raw_test_alert.created_at, + raw_test_alert.get('detected_at'), + raw_test_alert.get('created_at'), test_alert_data, ) %} {% do test_alerts.append(test_alert) %} From 8e7e947fa3691887fe7952f60493491348078007 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:36:30 +0000 Subject: [PATCH 08/15] Revert "Use .get() for all field accesses in populate_test_alerts to handle ClickHouse column names" This reverts commit 5a7efb8317bda1667397f3dbeb50123facfe9097. --- .../macros/alerts/population/test_alerts.sql | 76 +++++++++---------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql index 5e1b05cc5..31089372d 100644 --- a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql @@ -5,65 +5,63 @@ {% set raw_test_alerts_agate = run_query(elementary_cli.populate_test_alerts_query(days_back)) %} {% set raw_test_alerts = elementary.agate_to_dicts(raw_test_alerts_agate) %} {% for raw_test_alert in raw_test_alerts %} - {# ClickHouse may return original column names instead of aliases, so we use .get() for all field accesses #} - {% set alert_id = raw_test_alert.get('alert_id', raw_test_alert.get('id')) %} + {# ClickHouse may return original column names instead of aliases, so we handle both cases #} {% set test_type = raw_test_alert.get('alert_type', raw_test_alert.get('test_type')) %} - {% set status = raw_test_alert.get('status', '') | lower %} - {% set sub_type = raw_test_alert.get('sub_type', raw_test_alert.get('test_sub_type')) %} - {% set alert_description = raw_test_alert.get('alert_description', raw_test_alert.get('test_results_description')) %} - {% set alert_results_query = raw_test_alert.get('alert_results_query', raw_test_alert.get('test_results_query')) %} - {% set test_description = raw_test_alert.get('test_description', raw_test_alert.get('description')) %} - {% set test_meta = raw_test_alert.get('test_meta', raw_test_alert.get('meta')) %} - {% set model_meta = raw_test_alert.get('model_meta') %} - {% set result_rows = raw_test_alert.get('result_rows') %} + {% set status = raw_test_alert.status | lower %} + + {# ClickHouse may return original column names instead of aliases, so we handle both cases #} + {% set alert_id = raw_test_alert.get('alert_id', raw_test_alert.get('id')) %} {% set test_rows_sample = none %} {%- if not disable_samples and ((test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status != 'error')) -%} - {% set test_rows_sample = elementary_cli.get_test_rows_sample(result_rows, test_result_rows_agate.get(alert_id)) %} + {% set test_rows_sample = elementary_cli.get_test_rows_sample(raw_test_alert.result_rows, test_result_rows_agate.get(alert_id)) %} {%- endif -%} + {% set sub_type = raw_test_alert.get('sub_type', raw_test_alert.get('test_sub_type')) %} + {% set alert_description = raw_test_alert.get('alert_description', raw_test_alert.get('test_results_description')) %} + {% set alert_results_query = raw_test_alert.get('alert_results_query', raw_test_alert.get('test_results_query')) %} {% set test_alert_data = { 'id': alert_id, - 'alert_class_id': raw_test_alert.get('alert_class_id'), - 'model_unique_id': raw_test_alert.get('model_unique_id'), - 'test_unique_id': raw_test_alert.get('test_unique_id'), - 'detected_at': raw_test_alert.get('detected_at'), - 'database_name': raw_test_alert.get('database_name'), - 'schema_name': raw_test_alert.get('schema_name'), - 'table_name': raw_test_alert.get('table_name'), - 'column_name': raw_test_alert.get('column_name'), + 'alert_class_id': raw_test_alert.alert_class_id, + 'model_unique_id': raw_test_alert.model_unique_id, + 'test_unique_id': raw_test_alert.test_unique_id, + 'detected_at': raw_test_alert.detected_at, + 'database_name': raw_test_alert.database_name, + 'schema_name': raw_test_alert.schema_name, + 'table_name': raw_test_alert.table_name, + 'column_name': raw_test_alert.column_name, 'test_type': test_type, 'test_sub_type': sub_type, - 'test_description': test_description, + 'test_description': raw_test_alert.test_description, 'test_results_description': alert_description, - 'owners': raw_test_alert.get('owners'), - 'tags': raw_test_alert.get('tags'), + 'owners': raw_test_alert.owners, + 'tags': raw_test_alert.tags, 'test_results_query': alert_results_query, 'test_rows_sample': test_rows_sample, - 'other': raw_test_alert.get('other'), - 'test_name': raw_test_alert.get('test_name'), - 'test_short_name': raw_test_alert.get('test_short_name'), - 'test_params': raw_test_alert.get('test_params'), - 'severity': raw_test_alert.get('severity'), - 'test_meta': test_meta, - 'model_meta': model_meta, + 'other': raw_test_alert.other, + 'test_name': raw_test_alert.test_name, + 'test_short_name': raw_test_alert.test_short_name, + 'test_params': raw_test_alert.test_params, + 'severity': raw_test_alert.severity, + 'test_meta': raw_test_alert.test_meta, + 'model_meta': raw_test_alert.model_meta, 'status': status, - 'elementary_unique_id': raw_test_alert.get('elementary_unique_id'), - 'job_id': raw_test_alert.get('job_id'), - 'job_name': raw_test_alert.get('job_name'), - 'job_run_id': raw_test_alert.get('job_run_id'), - 'job_url': raw_test_alert.get('job_url'), - 'job_run_url': raw_test_alert.get('job_run_url'), - 'orchestrator': raw_test_alert.get('orchestrator') + 'elementary_unique_id': raw_test_alert.elementary_unique_id, + 'job_id': raw_test_alert.job_id, + 'job_name': raw_test_alert.job_name, + 'job_run_id': raw_test_alert.job_run_id, + 'job_url': raw_test_alert.job_url, + 'job_run_url': raw_test_alert.job_run_url, + 'orchestrator': raw_test_alert.orchestrator } %} {% set test_alert = elementary_cli.generate_alert_object( alert_id, - raw_test_alert.get('alert_class_id'), + raw_test_alert.alert_class_id, 'test', - raw_test_alert.get('detected_at'), - raw_test_alert.get('created_at'), + raw_test_alert.detected_at, + raw_test_alert.created_at, test_alert_data, ) %} {% do test_alerts.append(test_alert) %} From bb0d1590ab97fd3eb84c6bcbe1cd826cfdb5c0d6 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:36:48 +0000 Subject: [PATCH 09/15] Revert "Fix ClickHouse column alias handling in populate_test_alerts macro" This reverts commit a3bef9cf6897af76b604ac79f74e9c308b2ad6a3. --- .../macros/alerts/population/test_alerts.sql | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql index 31089372d..8ebe9eb0e 100644 --- a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql @@ -5,23 +5,16 @@ {% set raw_test_alerts_agate = run_query(elementary_cli.populate_test_alerts_query(days_back)) %} {% set raw_test_alerts = elementary.agate_to_dicts(raw_test_alerts_agate) %} {% for raw_test_alert in raw_test_alerts %} - {# ClickHouse may return original column names instead of aliases, so we handle both cases #} - {% set test_type = raw_test_alert.get('alert_type', raw_test_alert.get('test_type')) %} + {% set test_type = raw_test_alert.alert_type %} {% set status = raw_test_alert.status | lower %} - {# ClickHouse may return original column names instead of aliases, so we handle both cases #} - {% set alert_id = raw_test_alert.get('alert_id', raw_test_alert.get('id')) %} - {% set test_rows_sample = none %} {%- if not disable_samples and ((test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status != 'error')) -%} - {% set test_rows_sample = elementary_cli.get_test_rows_sample(raw_test_alert.result_rows, test_result_rows_agate.get(alert_id)) %} + {% set test_rows_sample = elementary_cli.get_test_rows_sample(raw_test_alert.result_rows, test_result_rows_agate.get(raw_test_alert.alert_id)) %} {%- endif -%} - {% set sub_type = raw_test_alert.get('sub_type', raw_test_alert.get('test_sub_type')) %} - {% set alert_description = raw_test_alert.get('alert_description', raw_test_alert.get('test_results_description')) %} - {% set alert_results_query = raw_test_alert.get('alert_results_query', raw_test_alert.get('test_results_query')) %} {% set test_alert_data = { - 'id': alert_id, + 'id': raw_test_alert.alert_id, 'alert_class_id': raw_test_alert.alert_class_id, 'model_unique_id': raw_test_alert.model_unique_id, 'test_unique_id': raw_test_alert.test_unique_id, @@ -31,12 +24,12 @@ 'table_name': raw_test_alert.table_name, 'column_name': raw_test_alert.column_name, 'test_type': test_type, - 'test_sub_type': sub_type, + 'test_sub_type': raw_test_alert.sub_type, 'test_description': raw_test_alert.test_description, - 'test_results_description': alert_description, + 'test_results_description': raw_test_alert.alert_description, 'owners': raw_test_alert.owners, 'tags': raw_test_alert.tags, - 'test_results_query': alert_results_query, + 'test_results_query': raw_test_alert.alert_results_query, 'test_rows_sample': test_rows_sample, 'other': raw_test_alert.other, 'test_name': raw_test_alert.test_name, @@ -53,11 +46,11 @@ 'job_url': raw_test_alert.job_url, 'job_run_url': raw_test_alert.job_run_url, 'orchestrator': raw_test_alert.orchestrator - } + } %} {% set test_alert = elementary_cli.generate_alert_object( - alert_id, + raw_test_alert.alert_id, raw_test_alert.alert_class_id, 'test', raw_test_alert.detected_at, From 1ce294e04310c3f39f96cf37d15484c4d9f5bc7a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:49:44 +0000 Subject: [PATCH 10/15] Add debug logging to identify Undefined field in populate_test_alerts Co-Authored-By: Itamar Hartstein --- .github/workflows/test-warehouse.yml | 1 + .../macros/alerts/population/test_alerts.sql | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index 71dd4dc36..17ab4ee49 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -200,6 +200,7 @@ jobs: - name: Run monitor env: SLACK_WEBHOOK: ${{ secrets.CI_SLACK_WEBHOOK }} + DBT_EDR_DEBUG: "1" run: > edr monitor -t "${{ inputs.warehouse-type }}" diff --git a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql index 8ebe9eb0e..0808e4e61 100644 --- a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql @@ -4,7 +4,50 @@ {% set test_alerts = [] %} {% set raw_test_alerts_agate = run_query(elementary_cli.populate_test_alerts_query(days_back)) %} {% set raw_test_alerts = elementary.agate_to_dicts(raw_test_alerts_agate) %} + + {# DEBUG: Log the number of alerts and column names from first row #} + {% do log('DEBUG populate_test_alerts: Processing ' ~ raw_test_alerts | length ~ ' alerts', info=True) %} + {% if raw_test_alerts | length > 0 %} + {% do log('DEBUG populate_test_alerts: Available columns in first row: ' ~ raw_test_alerts[0].keys() | list, info=True) %} + {% endif %} + {% for raw_test_alert in raw_test_alerts %} + {# DEBUG: Log each field access to identify which one is Undefined #} + {% do log('DEBUG populate_test_alerts: Processing alert index ' ~ loop.index0, info=True) %} + {% do log('DEBUG: alert_type = ' ~ raw_test_alert.alert_type, info=True) %} + {% do log('DEBUG: status = ' ~ raw_test_alert.status, info=True) %} + {% do log('DEBUG: alert_id = ' ~ raw_test_alert.alert_id, info=True) %} + {% do log('DEBUG: alert_class_id = ' ~ raw_test_alert.alert_class_id, info=True) %} + {% do log('DEBUG: model_unique_id = ' ~ raw_test_alert.model_unique_id, info=True) %} + {% do log('DEBUG: test_unique_id = ' ~ raw_test_alert.test_unique_id, info=True) %} + {% do log('DEBUG: detected_at = ' ~ raw_test_alert.detected_at, info=True) %} + {% do log('DEBUG: database_name = ' ~ raw_test_alert.database_name, info=True) %} + {% do log('DEBUG: schema_name = ' ~ raw_test_alert.schema_name, info=True) %} + {% do log('DEBUG: table_name = ' ~ raw_test_alert.table_name, info=True) %} + {% do log('DEBUG: column_name = ' ~ raw_test_alert.column_name, info=True) %} + {% do log('DEBUG: sub_type = ' ~ raw_test_alert.sub_type, info=True) %} + {% do log('DEBUG: test_description = ' ~ raw_test_alert.test_description, info=True) %} + {% do log('DEBUG: alert_description = ' ~ raw_test_alert.alert_description, info=True) %} + {% do log('DEBUG: owners = ' ~ raw_test_alert.owners, info=True) %} + {% do log('DEBUG: tags = ' ~ raw_test_alert.tags, info=True) %} + {% do log('DEBUG: alert_results_query = ' ~ raw_test_alert.alert_results_query, info=True) %} + {% do log('DEBUG: other = ' ~ raw_test_alert.other, info=True) %} + {% do log('DEBUG: test_name = ' ~ raw_test_alert.test_name, info=True) %} + {% do log('DEBUG: test_short_name = ' ~ raw_test_alert.test_short_name, info=True) %} + {% do log('DEBUG: test_params = ' ~ raw_test_alert.test_params, info=True) %} + {% do log('DEBUG: severity = ' ~ raw_test_alert.severity, info=True) %} + {% do log('DEBUG: test_meta = ' ~ raw_test_alert.test_meta, info=True) %} + {% do log('DEBUG: model_meta = ' ~ raw_test_alert.model_meta, info=True) %} + {% do log('DEBUG: elementary_unique_id = ' ~ raw_test_alert.elementary_unique_id, info=True) %} + {% do log('DEBUG: job_id = ' ~ raw_test_alert.job_id, info=True) %} + {% do log('DEBUG: job_name = ' ~ raw_test_alert.job_name, info=True) %} + {% do log('DEBUG: job_run_id = ' ~ raw_test_alert.job_run_id, info=True) %} + {% do log('DEBUG: job_url = ' ~ raw_test_alert.job_url, info=True) %} + {% do log('DEBUG: job_run_url = ' ~ raw_test_alert.job_run_url, info=True) %} + {% do log('DEBUG: orchestrator = ' ~ raw_test_alert.orchestrator, info=True) %} + {% do log('DEBUG: result_rows = ' ~ raw_test_alert.result_rows, info=True) %} + {% do log('DEBUG: created_at = ' ~ raw_test_alert.created_at, info=True) %} + {% set test_type = raw_test_alert.alert_type %} {% set status = raw_test_alert.status | lower %} @@ -49,6 +92,7 @@ } %} + {% do log('DEBUG: About to call generate_alert_object', info=True) %} {% set test_alert = elementary_cli.generate_alert_object( raw_test_alert.alert_id, raw_test_alert.alert_class_id, From a34bd77f67cbd42f186e371a64ff232e3427aaa7 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:58:26 +0000 Subject: [PATCH 11/15] Update debug logging to use 'is defined' checks for accurate Undefined detection Co-Authored-By: Itamar Hartstein --- .../macros/alerts/population/test_alerts.sql | 70 ++++++++++--------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql index 0808e4e61..54dfaa9fd 100644 --- a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql @@ -12,41 +12,41 @@ {% endif %} {% for raw_test_alert in raw_test_alerts %} - {# DEBUG: Log each field access to identify which one is Undefined #} + {# DEBUG: Use 'is defined' checks to accurately detect Undefined values #} {% do log('DEBUG populate_test_alerts: Processing alert index ' ~ loop.index0, info=True) %} - {% do log('DEBUG: alert_type = ' ~ raw_test_alert.alert_type, info=True) %} - {% do log('DEBUG: status = ' ~ raw_test_alert.status, info=True) %} - {% do log('DEBUG: alert_id = ' ~ raw_test_alert.alert_id, info=True) %} - {% do log('DEBUG: alert_class_id = ' ~ raw_test_alert.alert_class_id, info=True) %} - {% do log('DEBUG: model_unique_id = ' ~ raw_test_alert.model_unique_id, info=True) %} - {% do log('DEBUG: test_unique_id = ' ~ raw_test_alert.test_unique_id, info=True) %} - {% do log('DEBUG: detected_at = ' ~ raw_test_alert.detected_at, info=True) %} - {% do log('DEBUG: database_name = ' ~ raw_test_alert.database_name, info=True) %} - {% do log('DEBUG: schema_name = ' ~ raw_test_alert.schema_name, info=True) %} - {% do log('DEBUG: table_name = ' ~ raw_test_alert.table_name, info=True) %} - {% do log('DEBUG: column_name = ' ~ raw_test_alert.column_name, info=True) %} - {% do log('DEBUG: sub_type = ' ~ raw_test_alert.sub_type, info=True) %} - {% do log('DEBUG: test_description = ' ~ raw_test_alert.test_description, info=True) %} - {% do log('DEBUG: alert_description = ' ~ raw_test_alert.alert_description, info=True) %} - {% do log('DEBUG: owners = ' ~ raw_test_alert.owners, info=True) %} - {% do log('DEBUG: tags = ' ~ raw_test_alert.tags, info=True) %} - {% do log('DEBUG: alert_results_query = ' ~ raw_test_alert.alert_results_query, info=True) %} - {% do log('DEBUG: other = ' ~ raw_test_alert.other, info=True) %} - {% do log('DEBUG: test_name = ' ~ raw_test_alert.test_name, info=True) %} - {% do log('DEBUG: test_short_name = ' ~ raw_test_alert.test_short_name, info=True) %} - {% do log('DEBUG: test_params = ' ~ raw_test_alert.test_params, info=True) %} - {% do log('DEBUG: severity = ' ~ raw_test_alert.severity, info=True) %} - {% do log('DEBUG: test_meta = ' ~ raw_test_alert.test_meta, info=True) %} - {% do log('DEBUG: model_meta = ' ~ raw_test_alert.model_meta, info=True) %} - {% do log('DEBUG: elementary_unique_id = ' ~ raw_test_alert.elementary_unique_id, info=True) %} - {% do log('DEBUG: job_id = ' ~ raw_test_alert.job_id, info=True) %} - {% do log('DEBUG: job_name = ' ~ raw_test_alert.job_name, info=True) %} - {% do log('DEBUG: job_run_id = ' ~ raw_test_alert.job_run_id, info=True) %} - {% do log('DEBUG: job_url = ' ~ raw_test_alert.job_url, info=True) %} - {% do log('DEBUG: job_run_url = ' ~ raw_test_alert.job_run_url, info=True) %} - {% do log('DEBUG: orchestrator = ' ~ raw_test_alert.orchestrator, info=True) %} - {% do log('DEBUG: result_rows = ' ~ raw_test_alert.result_rows, info=True) %} - {% do log('DEBUG: created_at = ' ~ raw_test_alert.created_at, info=True) %} + {% do log('DEBUG: alert_type defined=' ~ (raw_test_alert.alert_type is defined) ~ ' val=' ~ raw_test_alert.alert_type, info=True) %} + {% do log('DEBUG: status defined=' ~ (raw_test_alert.status is defined) ~ ' val=' ~ raw_test_alert.status, info=True) %} + {% do log('DEBUG: alert_id defined=' ~ (raw_test_alert.alert_id is defined) ~ ' val=' ~ raw_test_alert.alert_id, info=True) %} + {% do log('DEBUG: alert_class_id defined=' ~ (raw_test_alert.alert_class_id is defined), info=True) %} + {% do log('DEBUG: model_unique_id defined=' ~ (raw_test_alert.model_unique_id is defined), info=True) %} + {% do log('DEBUG: test_unique_id defined=' ~ (raw_test_alert.test_unique_id is defined), info=True) %} + {% do log('DEBUG: detected_at defined=' ~ (raw_test_alert.detected_at is defined), info=True) %} + {% do log('DEBUG: database_name defined=' ~ (raw_test_alert.database_name is defined), info=True) %} + {% do log('DEBUG: schema_name defined=' ~ (raw_test_alert.schema_name is defined), info=True) %} + {% do log('DEBUG: table_name defined=' ~ (raw_test_alert.table_name is defined), info=True) %} + {% do log('DEBUG: column_name defined=' ~ (raw_test_alert.column_name is defined), info=True) %} + {% do log('DEBUG: sub_type defined=' ~ (raw_test_alert.sub_type is defined), info=True) %} + {% do log('DEBUG: test_description defined=' ~ (raw_test_alert.test_description is defined), info=True) %} + {% do log('DEBUG: alert_description defined=' ~ (raw_test_alert.alert_description is defined), info=True) %} + {% do log('DEBUG: owners defined=' ~ (raw_test_alert.owners is defined), info=True) %} + {% do log('DEBUG: tags defined=' ~ (raw_test_alert.tags is defined), info=True) %} + {% do log('DEBUG: alert_results_query defined=' ~ (raw_test_alert.alert_results_query is defined), info=True) %} + {% do log('DEBUG: other defined=' ~ (raw_test_alert.other is defined), info=True) %} + {% do log('DEBUG: test_name defined=' ~ (raw_test_alert.test_name is defined), info=True) %} + {% do log('DEBUG: test_short_name defined=' ~ (raw_test_alert.test_short_name is defined), info=True) %} + {% do log('DEBUG: test_params defined=' ~ (raw_test_alert.test_params is defined), info=True) %} + {% do log('DEBUG: severity defined=' ~ (raw_test_alert.severity is defined), info=True) %} + {% do log('DEBUG: test_meta defined=' ~ (raw_test_alert.test_meta is defined), info=True) %} + {% do log('DEBUG: model_meta defined=' ~ (raw_test_alert.model_meta is defined), info=True) %} + {% do log('DEBUG: elementary_unique_id defined=' ~ (raw_test_alert.elementary_unique_id is defined), info=True) %} + {% do log('DEBUG: job_id defined=' ~ (raw_test_alert.job_id is defined), info=True) %} + {% do log('DEBUG: job_name defined=' ~ (raw_test_alert.job_name is defined), info=True) %} + {% do log('DEBUG: job_run_id defined=' ~ (raw_test_alert.job_run_id is defined), info=True) %} + {% do log('DEBUG: job_url defined=' ~ (raw_test_alert.job_url is defined), info=True) %} + {% do log('DEBUG: job_run_url defined=' ~ (raw_test_alert.job_run_url is defined), info=True) %} + {% do log('DEBUG: orchestrator defined=' ~ (raw_test_alert.orchestrator is defined), info=True) %} + {% do log('DEBUG: result_rows defined=' ~ (raw_test_alert.result_rows is defined), info=True) %} + {% do log('DEBUG: created_at defined=' ~ (raw_test_alert.created_at is defined), info=True) %} {% set test_type = raw_test_alert.alert_type %} {% set status = raw_test_alert.status | lower %} @@ -92,6 +92,8 @@ } %} + {% do log('DEBUG: test_rows_sample = ' ~ test_rows_sample, info=True) %} + {% do log('DEBUG: test_alert_data keys = ' ~ test_alert_data.keys() | list, info=True) %} {% do log('DEBUG: About to call generate_alert_object', info=True) %} {% set test_alert = elementary_cli.generate_alert_object( raw_test_alert.alert_id, From f30af4fa583d29f9dbaeebc2ba13373c383fd040 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 17:32:21 +0000 Subject: [PATCH 12/15] Add more detailed debug logging to find nested Undefined values Co-Authored-By: Itamar Hartstein --- .../macros/alerts/population/test_alerts.sql | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql index 54dfaa9fd..12f0f02f9 100644 --- a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql @@ -94,6 +94,24 @@ {% do log('DEBUG: test_rows_sample = ' ~ test_rows_sample, info=True) %} {% do log('DEBUG: test_alert_data keys = ' ~ test_alert_data.keys() | list, info=True) %} + + {# DEBUG: Log actual VALUES of complex fields to find nested Undefined #} + {% do log('DEBUG VALUE: test_meta = ' ~ raw_test_alert.test_meta, info=True) %} + {% do log('DEBUG VALUE: model_meta = ' ~ raw_test_alert.model_meta, info=True) %} + {% do log('DEBUG VALUE: test_params = ' ~ raw_test_alert.test_params, info=True) %} + {% do log('DEBUG VALUE: owners = ' ~ raw_test_alert.owners, info=True) %} + {% do log('DEBUG VALUE: tags = ' ~ raw_test_alert.tags, info=True) %} + {% do log('DEBUG VALUE: other = ' ~ raw_test_alert.other, info=True) %} + {% do log('DEBUG VALUE: test_description = ' ~ raw_test_alert.test_description, info=True) %} + {% do log('DEBUG VALUE: alert_description = ' ~ raw_test_alert.alert_description, info=True) %} + + {# DEBUG: Check if any value in test_alert_data is Undefined using is defined #} + {% for key, value in test_alert_data.items() %} + {% if value is not defined %} + {% do log('DEBUG UNDEFINED FOUND: key=' ~ key, info=True) %} + {% endif %} + {% endfor %} + {% do log('DEBUG: About to call generate_alert_object', info=True) %} {% set test_alert = elementary_cli.generate_alert_object( raw_test_alert.alert_id, From c6103be2cbf37e457584f967bb04737638a7fc8d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 17:42:00 +0000 Subject: [PATCH 13/15] Fix ClickHouse column alias handling with .get() fallbacks ClickHouse returns columns with table prefix (e.g., 'failed_tests.database_name' instead of 'database_name'). Use .get() with fallbacks to handle both naming conventions for the affected fields: - database_name - schema_name - tags - test_params - severity - status - result_rows Co-Authored-By: Itamar Hartstein --- .../macros/alerts/population/test_alerts.sql | 80 +++---------------- 1 file changed, 9 insertions(+), 71 deletions(-) diff --git a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql index 12f0f02f9..745582334 100644 --- a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql @@ -5,65 +5,24 @@ {% set raw_test_alerts_agate = run_query(elementary_cli.populate_test_alerts_query(days_back)) %} {% set raw_test_alerts = elementary.agate_to_dicts(raw_test_alerts_agate) %} - {# DEBUG: Log the number of alerts and column names from first row #} - {% do log('DEBUG populate_test_alerts: Processing ' ~ raw_test_alerts | length ~ ' alerts', info=True) %} - {% if raw_test_alerts | length > 0 %} - {% do log('DEBUG populate_test_alerts: Available columns in first row: ' ~ raw_test_alerts[0].keys() | list, info=True) %} - {% endif %} - {% for raw_test_alert in raw_test_alerts %} - {# DEBUG: Use 'is defined' checks to accurately detect Undefined values #} - {% do log('DEBUG populate_test_alerts: Processing alert index ' ~ loop.index0, info=True) %} - {% do log('DEBUG: alert_type defined=' ~ (raw_test_alert.alert_type is defined) ~ ' val=' ~ raw_test_alert.alert_type, info=True) %} - {% do log('DEBUG: status defined=' ~ (raw_test_alert.status is defined) ~ ' val=' ~ raw_test_alert.status, info=True) %} - {% do log('DEBUG: alert_id defined=' ~ (raw_test_alert.alert_id is defined) ~ ' val=' ~ raw_test_alert.alert_id, info=True) %} - {% do log('DEBUG: alert_class_id defined=' ~ (raw_test_alert.alert_class_id is defined), info=True) %} - {% do log('DEBUG: model_unique_id defined=' ~ (raw_test_alert.model_unique_id is defined), info=True) %} - {% do log('DEBUG: test_unique_id defined=' ~ (raw_test_alert.test_unique_id is defined), info=True) %} - {% do log('DEBUG: detected_at defined=' ~ (raw_test_alert.detected_at is defined), info=True) %} - {% do log('DEBUG: database_name defined=' ~ (raw_test_alert.database_name is defined), info=True) %} - {% do log('DEBUG: schema_name defined=' ~ (raw_test_alert.schema_name is defined), info=True) %} - {% do log('DEBUG: table_name defined=' ~ (raw_test_alert.table_name is defined), info=True) %} - {% do log('DEBUG: column_name defined=' ~ (raw_test_alert.column_name is defined), info=True) %} - {% do log('DEBUG: sub_type defined=' ~ (raw_test_alert.sub_type is defined), info=True) %} - {% do log('DEBUG: test_description defined=' ~ (raw_test_alert.test_description is defined), info=True) %} - {% do log('DEBUG: alert_description defined=' ~ (raw_test_alert.alert_description is defined), info=True) %} - {% do log('DEBUG: owners defined=' ~ (raw_test_alert.owners is defined), info=True) %} - {% do log('DEBUG: tags defined=' ~ (raw_test_alert.tags is defined), info=True) %} - {% do log('DEBUG: alert_results_query defined=' ~ (raw_test_alert.alert_results_query is defined), info=True) %} - {% do log('DEBUG: other defined=' ~ (raw_test_alert.other is defined), info=True) %} - {% do log('DEBUG: test_name defined=' ~ (raw_test_alert.test_name is defined), info=True) %} - {% do log('DEBUG: test_short_name defined=' ~ (raw_test_alert.test_short_name is defined), info=True) %} - {% do log('DEBUG: test_params defined=' ~ (raw_test_alert.test_params is defined), info=True) %} - {% do log('DEBUG: severity defined=' ~ (raw_test_alert.severity is defined), info=True) %} - {% do log('DEBUG: test_meta defined=' ~ (raw_test_alert.test_meta is defined), info=True) %} - {% do log('DEBUG: model_meta defined=' ~ (raw_test_alert.model_meta is defined), info=True) %} - {% do log('DEBUG: elementary_unique_id defined=' ~ (raw_test_alert.elementary_unique_id is defined), info=True) %} - {% do log('DEBUG: job_id defined=' ~ (raw_test_alert.job_id is defined), info=True) %} - {% do log('DEBUG: job_name defined=' ~ (raw_test_alert.job_name is defined), info=True) %} - {% do log('DEBUG: job_run_id defined=' ~ (raw_test_alert.job_run_id is defined), info=True) %} - {% do log('DEBUG: job_url defined=' ~ (raw_test_alert.job_url is defined), info=True) %} - {% do log('DEBUG: job_run_url defined=' ~ (raw_test_alert.job_run_url is defined), info=True) %} - {% do log('DEBUG: orchestrator defined=' ~ (raw_test_alert.orchestrator is defined), info=True) %} - {% do log('DEBUG: result_rows defined=' ~ (raw_test_alert.result_rows is defined), info=True) %} - {% do log('DEBUG: created_at defined=' ~ (raw_test_alert.created_at is defined), info=True) %} - - {% set test_type = raw_test_alert.alert_type %} - {% set status = raw_test_alert.status | lower %} + {% set test_type = raw_test_alert.get('alert_type', raw_test_alert.get('failed_tests.test_type')) %} + {% set status = raw_test_alert.get('status', raw_test_alert.get('failed_tests.status')) | lower %} {% set test_rows_sample = none %} {%- if not disable_samples and ((test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status != 'error')) -%} - {% set test_rows_sample = elementary_cli.get_test_rows_sample(raw_test_alert.result_rows, test_result_rows_agate.get(raw_test_alert.alert_id)) %} + {% set test_rows_sample = elementary_cli.get_test_rows_sample(raw_test_alert.get('result_rows', raw_test_alert.get('failed_tests.result_rows')), test_result_rows_agate.get(raw_test_alert.alert_id)) %} {%- endif -%} + {# Use .get() with fallbacks for columns that ClickHouse may return with table prefix #} {% set test_alert_data = { 'id': raw_test_alert.alert_id, 'alert_class_id': raw_test_alert.alert_class_id, 'model_unique_id': raw_test_alert.model_unique_id, 'test_unique_id': raw_test_alert.test_unique_id, 'detected_at': raw_test_alert.detected_at, - 'database_name': raw_test_alert.database_name, - 'schema_name': raw_test_alert.schema_name, + 'database_name': raw_test_alert.get('database_name', raw_test_alert.get('failed_tests.database_name')), + 'schema_name': raw_test_alert.get('schema_name', raw_test_alert.get('failed_tests.schema_name')), 'table_name': raw_test_alert.table_name, 'column_name': raw_test_alert.column_name, 'test_type': test_type, @@ -71,14 +30,14 @@ 'test_description': raw_test_alert.test_description, 'test_results_description': raw_test_alert.alert_description, 'owners': raw_test_alert.owners, - 'tags': raw_test_alert.tags, + 'tags': raw_test_alert.get('tags', raw_test_alert.get('failed_tests.tags')), 'test_results_query': raw_test_alert.alert_results_query, 'test_rows_sample': test_rows_sample, 'other': raw_test_alert.other, 'test_name': raw_test_alert.test_name, 'test_short_name': raw_test_alert.test_short_name, - 'test_params': raw_test_alert.test_params, - 'severity': raw_test_alert.severity, + 'test_params': raw_test_alert.get('test_params', raw_test_alert.get('failed_tests.test_params')), + 'severity': raw_test_alert.get('severity', raw_test_alert.get('failed_tests.severity')), 'test_meta': raw_test_alert.test_meta, 'model_meta': raw_test_alert.model_meta, 'status': status, @@ -92,27 +51,6 @@ } %} - {% do log('DEBUG: test_rows_sample = ' ~ test_rows_sample, info=True) %} - {% do log('DEBUG: test_alert_data keys = ' ~ test_alert_data.keys() | list, info=True) %} - - {# DEBUG: Log actual VALUES of complex fields to find nested Undefined #} - {% do log('DEBUG VALUE: test_meta = ' ~ raw_test_alert.test_meta, info=True) %} - {% do log('DEBUG VALUE: model_meta = ' ~ raw_test_alert.model_meta, info=True) %} - {% do log('DEBUG VALUE: test_params = ' ~ raw_test_alert.test_params, info=True) %} - {% do log('DEBUG VALUE: owners = ' ~ raw_test_alert.owners, info=True) %} - {% do log('DEBUG VALUE: tags = ' ~ raw_test_alert.tags, info=True) %} - {% do log('DEBUG VALUE: other = ' ~ raw_test_alert.other, info=True) %} - {% do log('DEBUG VALUE: test_description = ' ~ raw_test_alert.test_description, info=True) %} - {% do log('DEBUG VALUE: alert_description = ' ~ raw_test_alert.alert_description, info=True) %} - - {# DEBUG: Check if any value in test_alert_data is Undefined using is defined #} - {% for key, value in test_alert_data.items() %} - {% if value is not defined %} - {% do log('DEBUG UNDEFINED FOUND: key=' ~ key, info=True) %} - {% endif %} - {% endfor %} - - {% do log('DEBUG: About to call generate_alert_object', info=True) %} {% set test_alert = elementary_cli.generate_alert_object( raw_test_alert.alert_id, raw_test_alert.alert_class_id, From 021fd2a0688f4ecd5fafb8d99a957e15bc4e699c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 17:48:07 +0000 Subject: [PATCH 14/15] Fix ClickHouse column name ambiguity with explicit SQL aliases ClickHouse returns columns with table prefix when column names are ambiguous across joined tables. The affected columns (database_name, schema_name, tags, test_params, severity, status, result_rows) exist in both failed_tests and tests tables. Fix: Add explicit 'as column_name' aliases in the SQL query to force ClickHouse to use the alias name instead of the qualified column name. This is cleaner than using .get() fallbacks in the macro. Co-Authored-By: Itamar Hartstein --- .../macros/alerts/population/test_alerts.sql | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql index 745582334..0ce7dc0da 100644 --- a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql @@ -6,23 +6,22 @@ {% set raw_test_alerts = elementary.agate_to_dicts(raw_test_alerts_agate) %} {% for raw_test_alert in raw_test_alerts %} - {% set test_type = raw_test_alert.get('alert_type', raw_test_alert.get('failed_tests.test_type')) %} - {% set status = raw_test_alert.get('status', raw_test_alert.get('failed_tests.status')) | lower %} + {% set test_type = raw_test_alert.alert_type %} + {% set status = raw_test_alert.status | lower %} {% set test_rows_sample = none %} {%- if not disable_samples and ((test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status != 'error')) -%} - {% set test_rows_sample = elementary_cli.get_test_rows_sample(raw_test_alert.get('result_rows', raw_test_alert.get('failed_tests.result_rows')), test_result_rows_agate.get(raw_test_alert.alert_id)) %} + {% set test_rows_sample = elementary_cli.get_test_rows_sample(raw_test_alert.result_rows, test_result_rows_agate.get(raw_test_alert.alert_id)) %} {%- endif -%} - {# Use .get() with fallbacks for columns that ClickHouse may return with table prefix #} {% set test_alert_data = { 'id': raw_test_alert.alert_id, 'alert_class_id': raw_test_alert.alert_class_id, 'model_unique_id': raw_test_alert.model_unique_id, 'test_unique_id': raw_test_alert.test_unique_id, 'detected_at': raw_test_alert.detected_at, - 'database_name': raw_test_alert.get('database_name', raw_test_alert.get('failed_tests.database_name')), - 'schema_name': raw_test_alert.get('schema_name', raw_test_alert.get('failed_tests.schema_name')), + 'database_name': raw_test_alert.database_name, + 'schema_name': raw_test_alert.schema_name, 'table_name': raw_test_alert.table_name, 'column_name': raw_test_alert.column_name, 'test_type': test_type, @@ -30,14 +29,14 @@ 'test_description': raw_test_alert.test_description, 'test_results_description': raw_test_alert.alert_description, 'owners': raw_test_alert.owners, - 'tags': raw_test_alert.get('tags', raw_test_alert.get('failed_tests.tags')), + 'tags': raw_test_alert.tags, 'test_results_query': raw_test_alert.alert_results_query, 'test_rows_sample': test_rows_sample, 'other': raw_test_alert.other, 'test_name': raw_test_alert.test_name, 'test_short_name': raw_test_alert.test_short_name, - 'test_params': raw_test_alert.get('test_params', raw_test_alert.get('failed_tests.test_params')), - 'severity': raw_test_alert.get('severity', raw_test_alert.get('failed_tests.severity')), + 'test_params': raw_test_alert.test_params, + 'severity': raw_test_alert.severity, 'test_meta': raw_test_alert.test_meta, 'model_meta': raw_test_alert.model_meta, 'status': status, @@ -140,25 +139,26 @@ failed_tests.test_execution_id, failed_tests.test_unique_id, failed_tests.model_unique_id, - failed_tests.database_name, + {# Explicit aliases for columns that exist in both failed_tests and tests tables to avoid ClickHouse column name ambiguity #} + failed_tests.database_name as database_name, failed_tests.detected_at, {{ elementary.edr_current_timestamp() }} as created_at, - failed_tests.schema_name, + failed_tests.schema_name as schema_name, failed_tests.table_name, failed_tests.column_name, failed_tests.test_type as alert_type, failed_tests.sub_type, failed_tests.test_results_description as alert_description, failed_tests.owners, - failed_tests.tags, + failed_tests.tags as tags, failed_tests.test_results_query as alert_results_query, failed_tests.other, failed_tests.test_name, failed_tests.test_short_name, - failed_tests.test_params, - failed_tests.severity, - failed_tests.status, - failed_tests.result_rows, + failed_tests.test_params as test_params, + failed_tests.severity as severity, + failed_tests.status as status, + failed_tests.result_rows as result_rows, tests.meta as test_meta, tests.description as test_description, artifacts_meta.meta as model_meta, From ef0a7244e910b0e1ca1202010169e16407f75e7d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 18:02:37 +0000 Subject: [PATCH 15/15] Remove DBT_EDR_DEBUG env var (debugging complete) Co-Authored-By: Itamar Hartstein --- .github/workflows/test-warehouse.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index 17ab4ee49..71dd4dc36 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -200,7 +200,6 @@ jobs: - name: Run monitor env: SLACK_WEBHOOK: ${{ secrets.CI_SLACK_WEBHOOK }} - DBT_EDR_DEBUG: "1" run: > edr monitor -t "${{ inputs.warehouse-type }}"