From c6cd0a671ace0b5ca92003f2be43571a495f0995 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 26 Feb 2026 22:03:29 +0000 Subject: [PATCH 1/2] refactor(topline): add source CTEs to int_topline__student_metrics to avoid repeated scans Five upstream tables were referenced multiple times across separate UNION ALL branches, causing BigQuery to scan each table more than once per model run: - int_topline__iready_diagnostic_weekly (2x) - int_topline__dibels_pm_weekly (2x) - int_topline__college_matriculation_weekly (4x) - int_topline__state_assessments_weekly (3x) - int_extracts__student_enrollments_weeks (3x) Adding named CTEs at the top of the with block for each of these and referencing the CTE alias in the body allows BigQuery to treat each as a single scan. https://claude.ai/code/session_01VwB9BpfUrxTnCVYPa16md9 --- .../int_topline__student_metrics.sql | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/src/dbt/kipptaf/models/topline/intermediate/int_topline__student_metrics.sql b/src/dbt/kipptaf/models/topline/intermediate/int_topline__student_metrics.sql index 6869da1cd4..4b1c78d5db 100644 --- a/src/dbt/kipptaf/models/topline/intermediate/int_topline__student_metrics.sql +++ b/src/dbt/kipptaf/models/topline/intermediate/int_topline__student_metrics.sql @@ -1,4 +1,22 @@ with + iready_diagnostic as ( + select * from {{ ref("int_topline__iready_diagnostic_weekly") }} + ), + + dibels_pm as (select * from {{ ref("int_topline__dibels_pm_weekly") }}), + + college_matriculation as ( + select * from {{ ref("int_topline__college_matriculation_weekly") }} + ), + + state_assessments as ( + select * from {{ ref("int_topline__state_assessments_weekly") }} + ), + + enrollments as ( + select * from {{ ref("int_extracts__student_enrollments_weeks") }} + ), + multilayer_metrics as ( /* K-8 Reading & Math */ select @@ -24,7 +42,7 @@ with is_proficient as metric_value, 'i-Ready Diagnostic' as indicator, - from {{ ref("int_topline__iready_diagnostic_weekly") }} + from iready_diagnostic union all @@ -52,7 +70,7 @@ with is_bfb_stretch_growth_int as metric_value, 'i-Ready B/FB Meeting Stretch Growth' as indicator, - from {{ ref("int_topline__iready_diagnostic_weekly") }} + from iready_diagnostic ), metric_union as ( @@ -109,7 +127,7 @@ with null as denominator, met_pm_round_overall_criteria as metric_value, - from {{ ref("int_topline__dibels_pm_weekly") }} + from dibels_pm union all @@ -127,7 +145,7 @@ with null as denominator, completed_test_round_int as metric_value, - from {{ ref("int_topline__dibels_pm_weekly") }} + from dibels_pm union all @@ -217,7 +235,7 @@ with null as denominator, is_proficient_int as metric_value, - from {{ ref("int_topline__state_assessments_weekly") }} + from state_assessments where region != 'Miami' union all @@ -236,7 +254,7 @@ with null as denominator, is_proficient_int as metric_value, - from {{ ref("int_topline__state_assessments_weekly") }} + from state_assessments where region = 'Miami' and test_round = 'PM3' union all @@ -255,7 +273,7 @@ with null as denominator, is_proficient_int as metric_value, - from {{ ref("int_topline__state_assessments_weekly") }} + from state_assessments where region = 'Miami' union all @@ -275,7 +293,7 @@ with null as denominator, if(is_enrolled_week, 1, 0) as metric_value, - from {{ ref("int_extracts__student_enrollments_weeks") }} + from enrollments union all @@ -293,7 +311,7 @@ with null as denominator, if(is_enrolled_week, 1, 0) as metric_value, - from {{ ref("int_extracts__student_enrollments_weeks") }} + from enrollments where not is_self_contained and not is_out_of_district union all @@ -508,7 +526,7 @@ with null as denominator, if(is_submitted_ba, 1, 0) as metric_value, - from {{ ref("int_topline__college_matriculation_weekly") }} + from college_matriculation union all @@ -526,7 +544,7 @@ with null as denominator, if(is_accepted_ba, 1, 0) as metric_value, - from {{ ref("int_topline__college_matriculation_weekly") }} + from college_matriculation union all @@ -544,7 +562,7 @@ with null as denominator, if(is_matriculated_ba, 1, 0) as metric_value, - from {{ ref("int_topline__college_matriculation_weekly") }} + from college_matriculation union all @@ -562,7 +580,7 @@ with null as denominator, is_submitted_quality_bar_4yr_int as metric_value, - from {{ ref("int_topline__college_matriculation_weekly") }} + from college_matriculation ) select @@ -582,7 +600,7 @@ select mu.numerator, mu.denominator, mu.metric_value, -from {{ ref("int_extracts__student_enrollments_weeks") }} as co +from enrollments as co left join metric_union as mu on co.student_number = mu.student_number From 2209a4d0b5e7e3d5122acb1b43e6a0e9cf61e538 Mon Sep 17 00:00:00 2001 From: Anthony Walters Date: Thu, 26 Feb 2026 22:19:51 +0000 Subject: [PATCH 2/2] trunk --- .../topline/intermediate/int_topline__student_metrics.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/dbt/kipptaf/models/topline/intermediate/int_topline__student_metrics.sql b/src/dbt/kipptaf/models/topline/intermediate/int_topline__student_metrics.sql index 4b1c78d5db..228de27be0 100644 --- a/src/dbt/kipptaf/models/topline/intermediate/int_topline__student_metrics.sql +++ b/src/dbt/kipptaf/models/topline/intermediate/int_topline__student_metrics.sql @@ -13,9 +13,7 @@ with select * from {{ ref("int_topline__state_assessments_weekly") }} ), - enrollments as ( - select * from {{ ref("int_extracts__student_enrollments_weeks") }} - ), + enrollments as (select * from {{ ref("int_extracts__student_enrollments_weeks") }}), multilayer_metrics as ( /* K-8 Reading & Math */