diff --git a/src/dbt/kippmiami/models/fldoe/intermediate/int_fldoe__all_assessments.sql b/src/dbt/kippmiami/models/fldoe/intermediate/int_fldoe__all_assessments.sql index bf1634a2e3..4b8600bcbd 100644 --- a/src/dbt/kippmiami/models/fldoe/intermediate/int_fldoe__all_assessments.sql +++ b/src/dbt/kippmiami/models/fldoe/intermediate/int_fldoe__all_assessments.sql @@ -23,24 +23,49 @@ with scale_score, achievement_level, is_proficient, + performance_level as achievement_level_int, cast( coalesce(assessment_grade, test_grade, enrolled_grade) as string ) as assessment_grade, - coalesce(performance_level, achievement_level_int) as performance_level, coalesce(student_id, fleid) as student_id, regexp_extract( _dbt_source_relation, r'stg_fldoe__(\w+)' ) as assessment_name, + from union_relations ) select * except (assessment_name), + 'Actual' as results_type, + 'KTAF FL' as district_state, + + administration_window as `admin`, + assessment_subject as `subject`, + if( assessment_name = 'science', 'Science', upper(assessment_name) ) as assessment_name, + + case + when assessment_subject like 'English Language Arts%' + then 'Text Study' + when assessment_subject in ('Algebra I', 'Algebra II', 'Geometry') + then 'Mathematics' + else assessment_subject + end as illuminate_subject, + + case + when achievement_level_int = 1 + then 'Below/Far Below' + when achievement_level_int = 2 + then 'Approaching' + when achievement_level_int >= 3 + then 'At/Above' + end as fast_aggregated_proficiency, + from transformed diff --git a/src/dbt/kippmiami/models/fldoe/intermediate/properties/int_fldoe__all_assessments.yml b/src/dbt/kippmiami/models/fldoe/intermediate/properties/int_fldoe__all_assessments.yml index 9afce84929..c8fa759767 100644 --- a/src/dbt/kippmiami/models/fldoe/intermediate/properties/int_fldoe__all_assessments.yml +++ b/src/dbt/kippmiami/models/fldoe/intermediate/properties/int_fldoe__all_assessments.yml @@ -21,9 +21,17 @@ models: data_type: boolean - name: assessment_grade data_type: string + - name: results_type + data_type: string + - name: district_state + data_type: string - name: performance_level data_type: int64 - name: student_id data_type: string - name: assessment_name data_type: string + - name: illuminate_subject + data_type: string + - name: fast_aggregated_proficiency + data_type: string diff --git a/src/dbt/kipptaf/macros/generate_cube_query.sql b/src/dbt/kipptaf/macros/generate_cube_query.sql deleted file mode 100644 index afd809cf18..0000000000 --- a/src/dbt/kipptaf/macros/generate_cube_query.sql +++ /dev/null @@ -1,117 +0,0 @@ -{% macro generate_cube_query( - dimensions, - metrics, - source_relation, - include_row_number=False, - focus_group=False, - focus_dims=[] -) %} - - select - - -- Dynamic dimensions - {% for dim in dimensions %} {{ dim }},{% endfor %} - - -- Dynamic metrics - {% for metric in metrics %} {{ metric }},{% endfor %} - - -- GROUPING() flags - {% for dim in dimensions %} - grouping({{ dim }}) as is_{{ dim }}_total, - {% endfor %} - - -- grouping_level using POW + GROUPING() - ( - {% for dim in dimensions %} - grouping({{ dim }}) * pow(2, {{ loop.index0 }}) - {% if not loop.last %} + {% endif %} - {% endfor %} - ) as grouping_level, - - -- focus_level label for active focus dimension - case - {% for dim in focus_dims %} - when - grouping({{ dim }}) = 0 - and {% for other in focus_dims if other != dim %} - grouping({{ other }}) = 1{% if not loop.last %} and {% endif %} - {% endfor %} - then '{{ dim }}' - {% endfor %} - when - {% for dim in focus_dims %} - grouping({{ dim }}) = 1{% if not loop.last %} and {% endif %} - {% endfor %} - then 'all_null' - else 'multi' - end as focus_level, - - -- total_type: hierarchical label with grouping_level - case - when - ( - {% for dim in dimensions %} - grouping({{ dim }}) * pow(2, {{ loop.index0 }}) - {% if not loop.last %} + {% endif %} - {% endfor %} - ) - = 0 - then 'Level 0: Detail' - - when - ( - {% for dim in dimensions %} - grouping({{ dim }}) * pow(2, {{ loop.index0 }}) - {% if not loop.last %} + {% endif %} - {% endfor %} - ) - = {{ 2 ** (dimensions | length) - 1 }} - then 'Level {{ 2 ** (dimensions | length) - 1 }}: Grand Total' - - else - concat( - 'Level ', - cast( - ( - {% for dim in dimensions %} - grouping({{ dim }}) * pow(2, {{ loop.index0 }}) - {% if not loop.last %} + {% endif %} - {% endfor %} - ) as string - ), - ': Subtotal – ', - array_to_string( - [ - {% for dim in dimensions %} - if(grouping({{ dim }}) = 1, '{{ dim }}', null) - {% if not loop.last %}, {% endif %} - {% endfor %} - ], - ', ' - ) - ) - end as total_type - - from {{ source_relation }} - - group by cube ({{ dimensions | join(", ") }}) - - {% if focus_group and focus_dims %} - having - ( - ( - {% for dim in focus_dims %} - cast(grouping({{ dim }}) = 0 as int64) - {% if not loop.last %} + {% endif %} - {% endfor %} - ) - = 1 - or ( - {% for dim in focus_dims %} - grouping({{ dim }}) = 1{% if not loop.last %} and {% endif %} - {% endfor %} - ) - ) - {% endif %} - -{% endmacro %} diff --git a/src/dbt/kipptaf/models/extracts/tableau/intermediate/int_tableau__state_assessments_demographic_comps.sql b/src/dbt/kipptaf/models/extracts/tableau/intermediate/int_tableau__state_assessments_demographic_comps.sql index 94a31064aa..1e9dd389a4 100644 --- a/src/dbt/kipptaf/models/extracts/tableau/intermediate/int_tableau__state_assessments_demographic_comps.sql +++ b/src/dbt/kipptaf/models/extracts/tableau/intermediate/int_tableau__state_assessments_demographic_comps.sql @@ -1,309 +1,311 @@ -with - assessment_scores as ( - select - _dbt_source_relation, - academic_year, - localstudentidentifier, - statestudentidentifier as state_id, - assessment_name, - is_proficient, - - 'Actual' as results_type, - 'KTAF NJ' as district_state, - - case - testcode - when 'SC05' - then 'SCI05' - when 'SC08' - then 'SCI08' - when 'SC11' - then 'SCI11' - else testcode - end as test_code, - - case - when race_ethnicity = 'B' - then 'African American' - when race_ethnicity = 'A' - then 'Asian' - when race_ethnicity = 'I' - then 'American Indian' - when race_ethnicity = 'H' - then 'Hispanic' - when race_ethnicity = 'P' - then 'Native Hawaiian' - when race_ethnicity = 'T' - then 'Other' - when race_ethnicity = 'W' - then 'White' - when race_ethnicity is null - then 'Blank' - end as aggregate_ethnicity, - - if(lep_status, 'ML', 'Not ML') as ml_status, - - if( - iep_status = 'Has IEP', - 'Students With Disabilities', - 'Students Without Disabilities' - ) as iep_status, - - from {{ ref("int_pearson__all_assessments") }} - where - testscalescore is not null and `period` = 'Spring' and academic_year >= 2018 - - union all - - select - _dbt_source_relation, - academic_year, - - null as localstudentidentifier, - - student_id as state_id, - assessment_name, - is_proficient, - - 'Actual' as results_type, - 'KTAF FL' as district_state, - - test_code, - - null as aggregate_ethnicity, - null as ml_status, - null as iep_status, - - from {{ ref("int_fldoe__all_assessments") }} - where scale_score is not null and season = 'Spring' - - union all - - select - _dbt_source_relation, - academic_year, - - null as localstudentidentifier, - - cast(state_student_identifier as string) as state_id, - - test_type as assessment_name, - - if( - performance_level - in ('Met Expectations', 'Exceeded Expectations', 'Graduation Ready'), - true, - false - ) as is_proficient, - - 'Preliminary' as results_type, - 'KTAF NJ' as district_state, - - case - when test_name = 'ELA Graduation Proficiency' - then 'ELAGP' - when test_name = 'Mathematics Graduation Proficiency' - then 'MATGP' - when test_name = 'Geometry' - then 'GEO01' - when test_name = 'Algebra I' - then 'ALG01' - when test_name like '%Mathematics%' - then concat('MAT', regexp_extract(test_name, r'.{6}(.{2})')) - when test_name like '%ELA%' - then concat('ELA', regexp_extract(test_name, r'.{6}(.{2})')) - end as test_code, - - null as aggregate_ethnicity, - null as ml_status, - null as iep_status, - - from {{ ref("stg_pearson__student_list_report") }} - where - state_student_identifier is not null - and administration = 'Spring' - and test_type = 'NJSLA' - and academic_year = {{ var("current_academic_year") }} - ) - -/* NJ scores */ -select - e.academic_year, - e.region, - e.student_number, - - a.district_state, - a.assessment_name, - a.aggregate_ethnicity, - a.ml_status, - a.iep_status, - - if( - e.lunch_status in ('F', 'R'), - 'Economically Disadvantaged', - 'Non Economically Disadvantaged' - ) as lunch_status, - - if(a.is_proficient, 1, 0) as is_proficient_int, - - case - when a.test_code = 'ALG01' - then concat(a.test_code, '_', e.school_level) - else a.test_code - end as test_code, - - case - e.gender when 'F' then 'Female' when 'M' then 'Male' when 'X' then 'Non-Binary' - end as gender, - -from {{ ref("int_extracts__student_enrollments") }} as e -inner join - assessment_scores as a - on e.academic_year = a.academic_year - and e.pearson_local_student_identifier = a.localstudentidentifier - and {{ union_dataset_join_clause(left_alias="e", right_alias="a") }} - and a.results_type = 'Actual' -where - e.rn_year = 1 - and e.academic_year >= {{ var("current_academic_year") - 7 }} - and e.grade_level > 2 - and e.school_level != 'OD' - -union all - -/* FL scores */ -select - e.academic_year, - e.region, - e.student_number, - - a.district_state, - a.assessment_name, - - case - when e.race_ethnicity = 'B' - then 'African American' - when e.race_ethnicity = 'A' - then 'Asian' - when e.race_ethnicity = 'I' - then 'American Indian' - when e.race_ethnicity = 'H' - then 'Hispanic' - when e.race_ethnicity = 'P' - then 'Native Hawaiian' - when e.race_ethnicity = 'T' - then 'Other' - when e.race_ethnicity = 'W' - then 'White' - when e.race_ethnicity is null - then 'Blank' - end as aggregate_ethnicity, - - e.ml_status, - - if( - e.iep_status = 'Has IEP', - 'Students With Disabilities', - 'Students Without Disabilities' - ) as iep_status, - - if( - e.lunch_status in ('F', 'R'), - 'Economically Disadvantaged', - 'Non Economically Disadvantaged' - ) as lunch_status, - - if(a.is_proficient, 1, 0) as is_proficient_int, - - case - when a.test_code = 'ALG01' - then concat(a.test_code, '_', e.school_level) - else a.test_code - end as test_code, - - case - e.gender when 'F' then 'Female' when 'M' then 'Male' when 'X' then 'Non-Binary' - end as gender, - -from {{ ref("int_extracts__student_enrollments") }} as e -inner join - assessment_scores as a - on e.academic_year = a.academic_year - and e.state_studentnumber = a.state_id - and {{ union_dataset_join_clause(left_alias="e", right_alias="a") }} - and a.results_type = 'Actual' -where - e.region = 'Miami' - and e.rn_year = 1 - and e.academic_year >= {{ var("current_academic_year") - 7 }} - and e.grade_level > 2 - - -- union all - /* NJ prelim scores */ - /* disabled until december -select - e.academic_year, - e.region, - e.student_number, - - a.district_state, - a.assessment_name, - - case - when e.race_ethnicity = 'B' - then 'African American' - when e.race_ethnicity = 'A' - then 'Asian' - when e.race_ethnicity = 'I' - then 'American Indian' - when e.race_ethnicity = 'H' - then 'Hispanic' - when e.race_ethnicity = 'P' - then 'Native Hawaiian' - when e.race_ethnicity = 'T' - then 'Other' - when e.race_ethnicity = 'W' - then 'White' - when e.race_ethnicity is null - then 'Blank' - end as aggregate_ethnicity, - - e.ml_status, - - if( - e.iep_status = 'Has IEP', - 'Students With Disabilities', - 'Students Without Disabilities' - ) as iep_status, - - if( - e.lunch_status in ('F', 'R'), - 'Economically Disadvantaged', - 'Non Economically Disadvantaged' - ) as lunch_status, - - if(a.is_proficient, 1, 0) as is_proficient_int, - - case - when a.test_code = 'ALG01' - then concat(a.test_code, '_', e.school_level) - else a.test_code - end as test_code, - - case - e.gender when 'F' then 'Female' when 'M' then 'Male' when 'X' then 'Non-Binary' - end as gender, - -from {{ ref("int_extracts__student_enrollments") }} as e -inner join - assessment_scores as a - on e.academic_year = a.academic_year - and e.state_studentnumber = a.state_id - and {{ union_dataset_join_clause(left_alias="e", right_alias="a") }} - and a.results_type = 'Preliminary' -where - e.academic_year = {{ var("current_academic_year") }} - and e.rn_year = 1 - and e.grade_level > 2 - and e.school_level != 'OD' -*/ +{# + Student-level assessment scores joined to enrollment demographics, + then aggregated via GROUPING SETS into demographic comparison rows. + + Each grouping set produces one demographic focus at a time (or a total), + crossed with region present-or-rolled-up — 12 sets total. +#} +{% set base_dims = [ + "academic_year", + "district_state", + "assessment_name", + "test_code", +] %} + +{% set focus_dims = [ + "gender", + "aggregate_ethnicity", + "lunch_status", + "ml_status", + "iep_status", +] %} + +with + /* + Prelim score gating: automatically includes preliminary NJ scores only + when official scores for that assessment/year have not yet landed in + int_pearson__all_assessments. This eliminates the need to manually + comment/uncomment the prelim branch each time a new student list file + is loaded — the branch self-deactivates once official scores arrive. + */ + prelim_assessments as ( + select academic_year, test_type, count(*) as record_count, + from {{ ref("int_pearson__student_list_report") }} + where + -- 2024: first year we track preliminary scores for comparison + academic_year >= 2024 + and administration = 'Spring' + and scale_score is not null + group by academic_year, test_type + ), + + valid_prelim_assessments as ( + select pa.academic_year, pa.test_type, + from prelim_assessments as pa + left join + {{ ref("int_pearson__all_assessments") }} as p + on pa.academic_year = p.academic_year + and pa.test_type = p.assessment_name + and p.season = 'Spring' + group by pa.academic_year, pa.test_type + having count(p.assessment_name) = 0 + ), + + test_code_metadata as ( + select + aligned_level_test_code, + any_value(school_level) as school_level, + any_value(grade_range_band) as grade_range_band, + any_value(discipline) as discipline, + from {{ ref("stg_google_sheets__state_test_comparison_demographics") }} + group by aligned_level_test_code + ), + + scores as ( + -- NJ's official scores + select + e.academic_year, + e.region, + e.student_number, + + a.district_state, + a.assessment_name, + a.is_proficient_int, + a.aligned_test_code as test_code, + a.aligned_ml_status as ml_status, + + e.aligned_gender as gender, + + a.aligned_aggregate_ethnicity as aggregate_ethnicity, + a.aligned_iep_status as iep_status, + + if( + e.lunch_status in ('F', 'R'), + 'Economically Disadvantaged', + 'Non Economically Disadvantaged' + ) as lunch_status, + + from {{ ref("int_extracts__student_enrollments") }} as e + inner join + {{ ref("int_pearson__all_assessments") }} as a + on e.academic_year = a.academic_year + and e.pearson_local_student_identifier = a.localstudentidentifier + and {{ union_dataset_join_clause(left_alias="e", right_alias="a") }} + and a.season = 'Spring' + and a.testscalescore is not null + where + e.rn_year = 1 + -- 2018: earliest year with available comps data + and e.academic_year >= 2018 + and e.grade_level > 2 + and e.school_level != 'OD' + + union all + + -- NJ's prelim scores + select + e.academic_year, + e.region, + e.student_number, + + a.district_state, + a.test_type as assessment_name, + a.is_proficient_int, + a.aligned_test_code as test_code, + + e.ml_status, + e.aligned_gender as gender, + + case + when e.race_ethnicity = 'B' + then 'African American' + when e.race_ethnicity = 'A' + then 'Asian' + when e.race_ethnicity = 'I' + then 'American Indian' + when e.race_ethnicity = 'H' + then 'Hispanic' + when e.race_ethnicity = 'P' + then 'Native Hawaiian' + when e.race_ethnicity = 'T' + then 'Other' + when e.race_ethnicity = 'W' + then 'White' + when e.race_ethnicity is null + then 'Blank' + end as aggregate_ethnicity, + + if( + e.iep_status = 'Has IEP', + 'Students With Disabilities', + 'Students Without Disabilities' + ) as iep_status, + + if( + e.lunch_status in ('F', 'R'), + 'Economically Disadvantaged', + 'Non Economically Disadvantaged' + ) as lunch_status, + + from {{ ref("int_extracts__student_enrollments") }} as e + inner join + {{ ref("int_pearson__student_list_report") }} as a + on e.academic_year = a.academic_year + and e.pearson_local_student_identifier = a.local_student_identifier + and {{ union_dataset_join_clause(left_alias="e", right_alias="a") }} + -- see prelim_assessments CTE + and a.academic_year >= 2024 + and a.administration = 'Spring' + and a.scale_score is not null + inner join + valid_prelim_assessments as vpa + on a.academic_year = vpa.academic_year + and a.test_type = vpa.test_type + where + e.rn_year = 1 + -- 2018: earliest year with available comps data + and e.academic_year >= 2018 + and e.grade_level > 2 + and e.school_level != 'OD' + + union all + + -- FL's official scores + select + e.academic_year, + e.region, + e.student_number, + + a.district_state, + a.assessment_name, + a.is_proficient_int, + a.test_code, + + e.ml_status, + e.aligned_gender as gender, + + case + when e.race_ethnicity = 'B' + then 'African American' + when e.race_ethnicity = 'A' + then 'Asian' + when e.race_ethnicity = 'I' + then 'American Indian' + when e.race_ethnicity = 'H' + then 'Hispanic' + when e.race_ethnicity = 'P' + then 'Native Hawaiian' + when e.race_ethnicity = 'T' + then 'Other' + when e.race_ethnicity = 'W' + then 'White' + when e.race_ethnicity is null + then 'Blank' + end as aggregate_ethnicity, + + if( + e.iep_status = 'Has IEP', + 'Students With Disabilities', + 'Students Without Disabilities' + ) as iep_status, + + if( + e.lunch_status in ('F', 'R'), + 'Economically Disadvantaged', + 'Non Economically Disadvantaged' + ) as lunch_status, + + from {{ ref("int_extracts__student_enrollments") }} as e + inner join + {{ ref("int_fldoe__all_assessments") }} as a + on e.academic_year = a.academic_year + and e.state_studentnumber = a.student_id + and {{ union_dataset_join_clause(left_alias="e", right_alias="a") }} + and a.results_type = 'Actual' + and a.scale_score is not null + and a.season = 'Spring' + where + e.region = 'Miami' + and e.rn_year = 1 + -- 2018: earliest year with available comps data + and e.academic_year >= 2018 + and e.grade_level > 2 + ) + +select + s.academic_year, + s.district_state, + s.region, + s.assessment_name, + s.test_code, + + round( + avg(s.is_proficient_int) * count(s.student_number), 0 + ) as total_proficient_students, + count(s.student_number) as total_students, + avg(s.is_proficient_int) as percent_proficient, + + /* (a) focus_level + demographic labels */ + case + {% for dim in focus_dims %} + when grouping({{ dim }}) = 0 then '{{ dim }}' + {% endfor %} + else 'all_null' + end as focus_level, + + case + when + {% for dim in focus_dims %} + grouping({{ dim }}) = 1{% if not loop.last %} and {% endif %} + {% endfor %} + then 'Total' + when + grouping(s.ml_status) = 0 + or grouping(s.iep_status) = 0 + or grouping(s.lunch_status) = 0 + then 'Subgroup' + when grouping(s.gender) = 0 + then 'Gender' + when grouping(s.aggregate_ethnicity) = 0 + then 'Aggregate Ethnicity' + end as comparison_demographic_group, + + case + when + {% for dim in focus_dims %} + grouping({{ dim }}) = 1{% if not loop.last %} and {% endif %} + {% endfor %} + then 'All Students' + else + coalesce( + s.gender, + s.aggregate_ethnicity, + s.lunch_status, + s.ml_status, + s.iep_status + ) + end as comparison_demographic_subgroup, + + /* (b) comparison_entity from region null-ness */ + if(grouping(s.region) = 1, s.district_state, 'Region') as comparison_entity, + + /* (c) test_code-derived columns via sheet lookup */ + any_value(m.school_level) as school_level, + any_value(m.grade_range_band) as grade_range_band, + any_value(m.discipline) as discipline, + +from scores as s +left join test_code_metadata as m on s.test_code = m.aligned_level_test_code +group by + grouping sets ( + {# Total (all focus dims rolled up) — with and without region #} + ({{ base_dims | join(", ") }}, s.region), + ({{ base_dims | join(", ") }}), + + {# One focus dim active at a time — with and without region #} + {% for dim in focus_dims %} + ({{ base_dims | join(", ") }}, s.region, {{ dim }}), + ({{ base_dims | join(", ") }}, {{ dim }}) + {% if not loop.last %},{% endif %} + {% endfor %} + ) diff --git a/src/dbt/kipptaf/models/extracts/tableau/intermediate/int_tableau__state_assessments_demographic_comps_cubed.sql b/src/dbt/kipptaf/models/extracts/tableau/intermediate/int_tableau__state_assessments_demographic_comps_cubed.sql deleted file mode 100644 index dea149cf8f..0000000000 --- a/src/dbt/kipptaf/models/extracts/tableau/intermediate/int_tableau__state_assessments_demographic_comps_cubed.sql +++ /dev/null @@ -1,34 +0,0 @@ -{% set dims = [ - "academic_year", - "district_state", - "region", - "assessment_name", - "test_code", - "gender", - "aggregate_ethnicity", - "lunch_status", - "ml_status", - "iep_status", -] %} - -{% set aggs = [ - "ROUND(AVG(is_proficient_int) * COUNT(student_number), 0) AS total_proficient_students", - "COUNT(student_number) AS total_students", - "AVG(is_proficient_int) AS percent_proficient", -] %} - -{{ - generate_cube_query( - dims, - aggs, - ref("int_tableau__state_assessments_demographic_comps"), - focus_group=True, - focus_dims=[ - "gender", - "aggregate_ethnicity", - "lunch_status", - "ml_status", - "iep_status", - ], - ) -}} diff --git a/src/dbt/kipptaf/models/extracts/tableau/intermediate/properties/int_tableau__state_assessments_demographic_comps.yml b/src/dbt/kipptaf/models/extracts/tableau/intermediate/properties/int_tableau__state_assessments_demographic_comps.yml index 116af090df..ac0eec342a 100644 --- a/src/dbt/kipptaf/models/extracts/tableau/intermediate/properties/int_tableau__state_assessments_demographic_comps.yml +++ b/src/dbt/kipptaf/models/extracts/tableau/intermediate/properties/int_tableau__state_assessments_demographic_comps.yml @@ -3,25 +3,45 @@ models: columns: - name: academic_year data_type: int64 - - name: region - data_type: string - - name: student_number - data_type: int64 - name: district_state data_type: string + - name: region + data_type: string - name: assessment_name data_type: string - - name: aggregate_ethnicity + - name: test_code data_type: string - - name: ml_status + - name: total_proficient_students + data_type: float64 + - name: total_students + data_type: int64 + - name: percent_proficient + data_type: float64 + - name: focus_level data_type: string - - name: iep_status + - name: comparison_demographic_group data_type: string - - name: lunch_status + - name: comparison_demographic_subgroup data_type: string - - name: is_proficient_int - data_type: int64 - - name: test_code + - name: comparison_entity + data_type: string + - name: school_level + data_type: string + - name: grade_range_band data_type: string - - name: gender + - name: discipline data_type: string + data_tests: + - dbt_utils.unique_combination_of_columns: + arguments: + combination_of_columns: + - academic_year + - district_state + - region + - assessment_name + - test_code + - comparison_entity + - comparison_demographic_group + - comparison_demographic_subgroup + config: + store_failures: true diff --git a/src/dbt/kipptaf/models/extracts/tableau/intermediate/properties/int_tableau__state_assessments_demographic_comps_cubed.yml b/src/dbt/kipptaf/models/extracts/tableau/intermediate/properties/int_tableau__state_assessments_demographic_comps_cubed.yml deleted file mode 100644 index caa8bab463..0000000000 --- a/src/dbt/kipptaf/models/extracts/tableau/intermediate/properties/int_tableau__state_assessments_demographic_comps_cubed.yml +++ /dev/null @@ -1,57 +0,0 @@ -models: - - name: int_tableau__state_assessments_demographic_comps_cubed - # config: - # materialized: table - columns: - - name: academic_year - data_type: int64 - - name: district_state - data_type: string - - name: region - data_type: string - - name: assessment_name - data_type: string - - name: test_code - data_type: string - - name: gender - data_type: string - - name: aggregate_ethnicity - data_type: string - - name: lunch_status - data_type: string - - name: ml_status - data_type: string - - name: iep_status - data_type: string - - name: total_proficient_students - data_type: float64 - - name: total_students - data_type: int64 - - name: percent_proficient - data_type: float64 - - name: is_academic_year_total - data_type: int64 - - name: is_district_state_total - data_type: int64 - - name: is_region_total - data_type: int64 - - name: is_assessment_name_total - data_type: int64 - - name: is_test_code_total - data_type: int64 - - name: is_gender_total - data_type: int64 - - name: is_aggregate_ethnicity_total - data_type: int64 - - name: is_lunch_status_total - data_type: int64 - - name: is_ml_status_total - data_type: int64 - - name: is_iep_status_total - data_type: int64 - - name: grouping_level - data_type: float64 - - name: focus_level - data_type: string - - name: total_type - data_type: string diff --git a/src/dbt/kipptaf/models/extracts/tableau/properties/rpt_tableau__state_assessments_dashboard_comps.yml b/src/dbt/kipptaf/models/extracts/tableau/properties/rpt_tableau__state_assessments_dashboard_comps.yml index e7695e561b..523673ff99 100644 --- a/src/dbt/kipptaf/models/extracts/tableau/properties/rpt_tableau__state_assessments_dashboard_comps.yml +++ b/src/dbt/kipptaf/models/extracts/tableau/properties/rpt_tableau__state_assessments_dashboard_comps.yml @@ -33,3 +33,17 @@ models: data_type: boolean - name: region_matched_or_outperformed data_type: boolean + data_tests: + - dbt_utils.unique_combination_of_columns: + arguments: + combination_of_columns: + - academic_year + - school_level + - assessment_name + - test_code + - region + - comparison_entity + - comparison_demographic_group + - comparison_demographic_subgroup + config: + store_failures: true diff --git a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__state_assessments_dashboard.sql b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__state_assessments_dashboard.sql index d4478c4931..af09cb44e5 100644 --- a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__state_assessments_dashboard.sql +++ b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__state_assessments_dashboard.sql @@ -70,10 +70,10 @@ with state_comps as ( select academic_year, - test_name, - test_code, + assessment_name, + aligned_test_code, region, - 'Spring' as season, + season, {% for entity in comparison_entities %} avg( @@ -91,8 +91,11 @@ with {% if not loop.last %},{% endif %} {% endfor %} - from {{ ref("stg_google_sheets__state_test_comparison") }} - group by academic_year, test_name, test_code, region + from {{ ref("stg_google_sheets__state_test_comparison_demographics") }} + where + comparison_demographic_group = 'Total' + and comparison_demographic_subgroup = 'All Students' + group by academic_year, assessment_name, aligned_test_code, region, season ), assessment_scores as ( @@ -113,28 +116,12 @@ with race_ethnicity, test_grade, - 'Actual' as results_type, - - if(`period` = 'FallBlock', 'Fall', `period`) as `admin`, - - if(`period` = 'FallBlock', 'Fall', `period`) as season, + results_type, - if( - `subject` = 'English Language Arts/Literacy', - 'English Language Arts', - `subject` - ) as `subject`, - - case - testcode - when 'SC05' - then 'SCI05' - when 'SC08' - then 'SCI08' - when 'SC11' - then 'SCI11' - else testcode - end as test_code, + `admin`, + season, + aligned_subject as `subject`, + aligned_test_code as test_code, from {{ ref("int_pearson__all_assessments") }} where @@ -225,7 +212,7 @@ with null as race_ethnicity, null as test_grade, - 'Preliminary' as results_type, + results_type, administration as `admin`, administration as season, @@ -237,22 +224,9 @@ with else 'English Language Arts' end as subject, - case - when test_name = 'ELA Graduation Proficiency' - then 'ELAGP' - when test_name = 'Mathematics Graduation Proficiency' - then 'MATGP' - when test_name = 'Geometry' - then 'GEO01' - when test_name = 'Algebra I' - then 'ALG01' - when test_name like '%Mathematics%' - then concat('MAT', regexp_extract(test_name, r'.{6}(.{2})')) - when test_name like '%ELA%' - then concat('ELA', regexp_extract(test_name, r'.{6}(.{2})')) - end as test_code, + aligned_test_code as test_code, - from {{ ref("stg_pearson__student_list_report") }} + from {{ ref("int_pearson__student_list_report") }} where state_student_identifier is not null and administration = 'Spring' @@ -347,8 +321,8 @@ inner join left join state_comps as c on a.academic_year = c.academic_year - and a.assessment_name = c.test_name - and a.test_code = c.test_code + and a.assessment_name = c.assessment_name + and a.test_code = c.aligned_test_code and a.season = c.season and e.region = c.region left join @@ -467,8 +441,8 @@ inner join left join state_comps as c on a.academic_year = c.academic_year - and a.assessment_name = c.test_name - and a.test_code = c.test_code + and a.assessment_name = c.assessment_name + and a.test_code = c.aligned_test_code and a.season = c.season and e.region = c.region left join @@ -586,8 +560,8 @@ inner join left join state_comps as c on a.academic_year = c.academic_year - and a.assessment_name = c.test_name - and a.test_code = c.test_code + and a.assessment_name = c.assessment_name + and a.test_code = c.aligned_test_code and a.season = c.season and e.region = c.region left join diff --git a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__state_assessments_dashboard_comps.sql b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__state_assessments_dashboard_comps.sql index 8967aa7aa1..56ea5678cb 100644 --- a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__state_assessments_dashboard_comps.sql +++ b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__state_assessments_dashboard_comps.sql @@ -1,333 +1,149 @@ -with - -- trunk-ignore(sqlfluff/ST03) - ktaf as ( - select - b.academic_year, - b.assessment_name, - b.test_code, - b.total_proficient_students, - b.total_students, - b.percent_proficient, - b.focus_level, - - if(b.region is null, regions, b.region) as region, - - case - when b.focus_level = 'all_null' - then 'Total' - when b.focus_level in ('ml_status', 'iep_status', 'lunch_status') - then 'Subgroup' - else initcap(regexp_replace(b.focus_level, r'_', ' ')) - end as comparison_demographic_group, - - case - when b.focus_level = 'all_null' - then 'All Students' - else - coalesce( - b.gender, - b.aggregate_ethnicity, - b.lunch_status, - b.ml_status, - b.iep_status - ) - end as comparison_demographic_subgroup, - - if(b.region is null, b.district_state, 'Region') as comparison_entity, - - from {{ ref("int_tableau__state_assessments_demographic_comps_cubed") }} as b - cross join unnest(['Camden', 'Newark']) as regions - where - b.academic_year is not null - and b.assessment_name is not null - and b.test_code is not null - and b.district_state = 'KTAF NJ' - - union all - - select - academic_year, - assessment_name, - test_code, - total_proficient_students, - total_students, - percent_proficient, - focus_level, - - coalesce(region, 'Miami') as region, - - case - when focus_level = 'all_null' - then 'Total' - when focus_level in ('ml_status', 'iep_status', 'lunch_status') - then 'Subgroup' - else initcap(regexp_replace(focus_level, r'_', ' ')) - end as comparison_demographic_group, - - case - when focus_level = 'all_null' - then 'All Students' - else - coalesce( - gender, aggregate_ethnicity, lunch_status, ml_status, iep_status - ) - end as comparison_demographic_subgroup, - - if(region is null, district_state, 'Region') as comparison_entity, - - from {{ ref("int_tableau__state_assessments_demographic_comps_cubed") }} - where - academic_year is not null - and assessment_name is not null - and test_code is not null - and district_state = 'KTAF FL' - ), - - -- deduping here because of how group by cube generates rows - dedup_ktaf as ( - {{ - dbt_utils.deduplicate( - relation="ktaf", - partition_by="academic_year, region, comparison_entity,comparison_demographic_group,comparison_demographic_subgroup,focus_level,assessment_name,test_code", - order_by="academic_year", - ) - }} - ), - - appended as ( - select - academic_year, - assessment_name, - region, - comparison_entity, - focus_level, - - total_students, - percent_proficient, - - comparison_demographic_group, - comparison_demographic_subgroup, - - total_proficient_students, - - if(test_code like 'ALG01%', 'ALG01', test_code) as test_code, - - case - when - test_code in ( - 'ELA09', - 'ELA10', - 'ELA11', - 'ELAGP', - 'ALG01_HS', - 'GEO01', - 'ALG02', - 'MATGP', - 'SCI11' - ) - then 'HS' - when test_code = 'ALG01_MS' - then 'MS' - when safe_cast(right(test_code, 2) as numeric) between 5 and 8 - then 'MS' - else 'ES' - end as school_level, - - case - when - test_code in ( - 'ELA09', - 'ELA10', - 'ELA11', - 'ELAGP', - 'ALG01_HS', - 'GEO01', - 'ALG02', - 'MATGP', - 'SCI11' - ) - then 'HS' - else '3-8' - end as grade_range_band, - - case - when left(test_code, 3) in ('MAT', 'ALG', 'GEO') - then 'Math' - when left(test_code, 3) = 'ELA' - then 'ELA' - when left(test_code, 3) = 'SCI' - then 'Science' - when left(test_code, 3) = 'SOC' - then 'Social Studies' - end as discipline, - - from dedup_ktaf - where - comparison_demographic_subgroup - not in ('Not ML', 'Students Without Disabilities', 'Non-Binary') - - union all - - select - academic_year, - test_name as assessment_name, - region, - comparison_entity, - null as focus_level, - - total_students, - percent_proficient, - - if( - comparison_demographic_subgroup - in ('Grade - 08', 'Grade - 09', 'Grade - 10'), - 'Total', - comparison_demographic_group - ) as comparison_demographic_group, - - if( - comparison_demographic_group = 'Grade', - 'All Students', - comparison_demographic_subgroup - ) as comparison_demographic_subgroup, - - round(percent_proficient * total_students, 0) as total_proficient_students, - - test_code, - - case - when - comparison_demographic_subgroup = 'Grade - 08' - and test_code = 'ALG01' - then 'MS' - when comparison_demographic_subgroup in ('Grade - 09', 'Grade - 10') - then 'HS' - when - test_code in ( - 'ELA09', - 'ELA10', - 'ELA11', - 'ELAGP', - 'ALG01', - 'GEO01', - 'ALG02', - 'MATGP', - 'SCI11' - ) - then 'HS' - when safe_cast(right(test_code, 2) as numeric) between 5 and 8 - then 'MS' - else 'ES' - end as school_level, - - case - when comparison_demographic_subgroup in ('Grade - 09', 'Grade - 10') - then 'HS' - when - test_code in ( - 'ELA09', - 'ELA10', - 'ELA11', - 'ELAGP', - 'ALG02', - 'GEO01', - 'MATGP', - 'SCI11' - ) - then 'HS' - else '3-8' - end as grade_range_band, - - case - when left(test_code, 3) in ('MAT', 'ALG', 'GEO') - then 'Math' - when left(test_code, 3) = 'ELA' - then 'ELA' - when left(test_code, 3) = 'SCI' - then 'Science' - when left(test_code, 3) = 'SOC' - then 'Social Studies' - end as discipline, - - from {{ ref("stg_google_sheets__state_test_comparison_demographics") }} - where comparison_demographic_subgroup != 'SE Accommodation' - ), - - grouped_comps as ( - select - academic_year, - school_level, - grade_range_band, - assessment_name, - discipline, - test_code, - region, - comparison_entity, - comparison_demographic_group, - comparison_demographic_subgroup, - focus_level, - - sum(total_proficient_students) as total_proficient_students, - - sum(total_students) as total_students, - - safe_divide( - sum(total_proficient_students), sum(total_students) - ) as percent_proficient, - - from appended - where comparison_demographic_subgroup != 'Blank' - group by - academic_year, - school_level, - grade_range_band, - assessment_name, - discipline, - test_code, - region, - comparison_entity, - comparison_demographic_group, - comparison_demographic_subgroup, - focus_level - ) - -select - a.academic_year, - a.school_level, - a.grade_range_band, - a.assessment_name, - a.discipline, - a.test_code, - a.region, - a.comparison_entity, - a.comparison_demographic_group, - a.comparison_demographic_subgroup, - a.total_proficient_students, - a.total_students, - a.percent_proficient, - - if(b.percent_proficient = a.percent_proficient, true, false) as region_matched, - if(b.percent_proficient > a.percent_proficient, true, false) as region_outperformed, - - if( - b.percent_proficient = a.percent_proficient - or b.percent_proficient > a.percent_proficient, - true, - false - ) as region_matched_or_outperformed, - -from grouped_comps as a -left join - grouped_comps as b - on a.academic_year = b.academic_year - and a.academic_year = b.academic_year - and a.school_level = b.school_level - and a.grade_range_band = b.grade_range_band - and a.assessment_name = b.assessment_name - and a.discipline = b.discipline - and a.test_code = b.test_code - and a.region = b.region - and a.comparison_entity = b.comparison_entity - and a.comparison_demographic_group = b.comparison_demographic_group - and a.comparison_demographic_subgroup = b.comparison_demographic_subgroup - and b.comparison_entity = 'Region' +with + appended as ( + /* NJ: cross join fans out district-level rows to Camden + Newark */ + select + academic_year, + assessment_name, + comparison_entity, + comparison_demographic_group, + comparison_demographic_subgroup, + focus_level, + school_level, + grade_range_band, + discipline, + total_proficient_students, + total_students, + percent_proficient, + + if(test_code like 'ALG01%', 'ALG01', test_code) as test_code, + if(region is null, regions, region) as region, + + from {{ ref("int_tableau__state_assessments_demographic_comps") }} + cross join unnest(['Camden', 'Newark']) as regions + where + district_state = 'KTAF NJ' + and (region is null or region = regions) + and comparison_demographic_subgroup + not in ('Not ML', 'Students Without Disabilities', 'Non-Binary', 'Blank') + + union all + + /* FL */ + select + academic_year, + assessment_name, + comparison_entity, + comparison_demographic_group, + comparison_demographic_subgroup, + focus_level, + school_level, + grade_range_band, + discipline, + total_proficient_students, + total_students, + percent_proficient, + + if(test_code like 'ALG01%', 'ALG01', test_code) as test_code, + coalesce(region, 'Miami') as region, + + from {{ ref("int_tableau__state_assessments_demographic_comps") }} + where + district_state = 'KTAF FL' + and comparison_demographic_subgroup + not in ('Not ML', 'Students Without Disabilities', 'Non-Binary', 'Blank') + + union all + + /* Google Sheets benchmarks */ + select + academic_year, + assessment_name, + comparison_entity, + aligned_comparison_demographic_group as comparison_demographic_group, + aligned_comparison_demographic_subgroup as comparison_demographic_subgroup, + null as focus_level, + school_level, + grade_range_band, + discipline, + total_proficient_students, + total_students, + percent_proficient, + aligned_test_code as test_code, + region, + + from {{ ref("stg_google_sheets__state_test_comparison_demographics") }} + where comparison_demographic_subgroup not in ('SE Accommodation', 'Blank') + ), + + grouped_comps as ( + select + academic_year, + school_level, + grade_range_band, + assessment_name, + discipline, + test_code, + region, + comparison_entity, + comparison_demographic_group, + comparison_demographic_subgroup, + focus_level, + + sum(total_proficient_students) as total_proficient_students, + + sum(total_students) as total_students, + + safe_divide( + sum(total_proficient_students), sum(total_students) + ) as percent_proficient, + + from appended + group by + academic_year, + school_level, + grade_range_band, + assessment_name, + discipline, + test_code, + region, + comparison_entity, + comparison_demographic_group, + comparison_demographic_subgroup, + focus_level + ) + +select + a.academic_year, + a.school_level, + a.grade_range_band, + a.assessment_name, + a.discipline, + a.test_code, + a.region, + a.comparison_entity, + a.comparison_demographic_group, + a.comparison_demographic_subgroup, + a.total_proficient_students, + a.total_students, + a.percent_proficient, + + if(b.percent_proficient = a.percent_proficient, true, false) as region_matched, + if(b.percent_proficient > a.percent_proficient, true, false) as region_outperformed, + + if( + b.percent_proficient >= a.percent_proficient, true, false + ) as region_matched_or_outperformed, + +from grouped_comps as a +left join + grouped_comps as b + on a.academic_year = b.academic_year + and a.school_level = b.school_level + and a.grade_range_band = b.grade_range_band + and a.assessment_name = b.assessment_name + and a.discipline = b.discipline + and a.test_code = b.test_code + and a.region = b.region + and a.comparison_demographic_group = b.comparison_demographic_group + and a.comparison_demographic_subgroup = b.comparison_demographic_subgroup + and b.comparison_entity = 'Region' diff --git a/src/dbt/kipptaf/models/fldoe/intermediate/int_fldoe__all_assessments.sql b/src/dbt/kipptaf/models/fldoe/intermediate/int_fldoe__all_assessments.sql index 1cda1266d4..528bd120e7 100644 --- a/src/dbt/kipptaf/models/fldoe/intermediate/int_fldoe__all_assessments.sql +++ b/src/dbt/kipptaf/models/fldoe/intermediate/int_fldoe__all_assessments.sql @@ -58,8 +58,38 @@ select cw1.sublevel_number, cw1.sublevel_name, + 'Actual' as results_type, + 'KTAF FL' as district_state, + cast(regexp_extract(fl.achievement_level, r'\d+') as int) as achievement_level_int, + case + when fl.test_code = 'ALG01' and fl.assessment_grade = '8' + then concat(fl.test_code, '_', 'MS') + when fl.test_code = 'ALG01' and fl.assessment_grade in ('9', '10', '11', '12') + then concat(fl.test_code, '_', 'HS') + else fl.test_code + end as aligned_level_test_code, + + case + when fl.assessment_subject like 'English Language Arts%' + then 'Text Study' + when fl.assessment_subject in ('Algebra I', 'Algebra II', 'Geometry') + then 'Mathematics' + else fl.assessment_subject + end as illuminate_subject, + + case + when cast(regexp_extract(fl.achievement_level, r'\d+') as int) = 1 + then 'Below/Far Below' + when cast(regexp_extract(fl.achievement_level, r'\d+') as int) = 2 + then 'Approaching' + when cast(regexp_extract(fl.achievement_level, r'\d+') as int) >= 3 + then 'At/Above' + end as fast_aggregated_proficiency, + + if(fl.is_proficient, 1, 0) as is_proficient_int, + if(cw1.sublevel_number >= 6, null, cw2.scale_low) as scale_for_proficiency, if( @@ -76,6 +106,7 @@ select partition by fl.student_id, fl.academic_year, fl.assessment_subject order by fl.administration_window asc ) as scale_score_prev, + from source as fl left join scale_crosswalk as sc diff --git a/src/dbt/kipptaf/models/fldoe/intermediate/properties/int_fldoe__all_assessments.yml b/src/dbt/kipptaf/models/fldoe/intermediate/properties/int_fldoe__all_assessments.yml index ec9b5311aa..cd3711f071 100644 --- a/src/dbt/kipptaf/models/fldoe/intermediate/properties/int_fldoe__all_assessments.yml +++ b/src/dbt/kipptaf/models/fldoe/intermediate/properties/int_fldoe__all_assessments.yml @@ -1,47 +1,163 @@ models: - name: int_fldoe__all_assessments + description: > + Network-level union of FLDOE FAST assessment results from the kippmiami + district project, enriched with iReady scale crosswalk data for sublevel + classification and growth/proficiency targets. Adds standardized metadata + columns (results_type, district_state, illuminate_subject, + fast_aggregated_proficiency) and derived score columns for use in + reporting and demographic comparisons. + data_tests: + - dbt_utils.unique_combination_of_columns: + arguments: + combination_of_columns: + - _dbt_source_relation + - student_id + - academic_year + - administration_window + - test_code columns: - name: _dbt_source_relation data_type: string + description: Source relation identifier from dbt_utils.union_relations. + - name: test_code data_type: string + description: State assessment test code (e.g. ELA03, ALG01, ELAGP). + - name: academic_year data_type: int64 + description: Academic year the assessment was administered. + - name: administration_window data_type: string + description: Assessment administration window (PM1, PM2, or PM3). + - name: season data_type: string + description: > + Testing season derived from administration window (Fall, Winter, + Spring). We only use Spring for official reporting. + - name: discipline data_type: string + description: Subject discipline category (Math, ELA, Science, etc.). + - name: assessment_subject data_type: string + description: Full assessment subject name as reported by FLDOE. + - name: scale_score data_type: int64 + description: Student's raw scale score on the assessment. + - name: achievement_level data_type: string + description: > + Achievement level label as reported by FLDOE (e.g. 'Level 1', 'Level + 3'). + - name: is_proficient data_type: boolean + description: + Whether the student scored at or above the proficiency threshold. + - name: assessment_grade data_type: string + description: > + Grade level of the assessment. Ideally, it should match a student's + enrolled grade. + - name: performance_level data_type: int64 + description: Numeric performance level from the source data. + - name: student_id data_type: string + description: Student identifier from the source system. + - name: assessment_name data_type: string + description: Full assessment name (e.g. FAST, FSA, EOC). + - name: sublevel_number data_type: int64 + description: > + iReady crosswalk sublevel number for the student's scale score band; + null if no crosswalk match. + - name: sublevel_name data_type: string + description: > + iReady crosswalk sublevel name for the student's scale score band; + null if no crosswalk match. + + - name: results_type + data_type: string + description: Always 'Actual' for FLDOE results (no preliminary scores). + + - name: district_state + data_type: string + description: + Network district/state identifier; always 'KTAF FL' for this model. + - name: achievement_level_int data_type: int64 + description: > + Numeric achievement level extracted from the achievement_level label + (e.g. 3 from 'Level 3'). + + - name: aligned_level_test_code + data_type: string + description: > + Test code adjusted for school level — ALG01 grade 8 is suffixed with + '_MS', grades 9-12 with '_HS'; all other test codes pass through + unchanged. + + - name: illuminate_subject + data_type: string + description: > + Standardized subject label for Illuminate/reporting — ELA subjects map + to 'Text Study'; Algebra I, Algebra II, and Geometry map to + 'Mathematics'; all others pass through unchanged. + + - name: fast_aggregated_proficiency + data_type: string + description: > + Aggregated proficiency band: level 1 = 'Below/Far Below', level 2 = + 'Approaching', level 3+ = 'At/Above'. + + - name: is_proficient_int + data_type: int64 + description: + Integer representation of is_proficient (1 = true, 0 = false). + - name: scale_for_proficiency data_type: int64 + description: > + Minimum scale score required to reach proficiency (sublevel 6) for + this subject and grade; null if already at or above sublevel 6. + - name: points_to_proficiency data_type: int64 + description: > + Points needed to reach proficiency (scale_for_proficiency minus + scale_score); null if already at or above sublevel 6. + - name: scale_for_growth data_type: int64 + description: > + Minimum scale score for the next sublevel (scale_high + 1 of current + sublevel); null if already at the highest sublevel (8). + - name: points_to_growth data_type: int64 + description: > + Points needed to reach the next sublevel (scale_for_growth minus + scale_score); null if already at the highest sublevel (8). + - name: scale_score_prev data_type: int64 + description: > + Student's scale score from the prior administration window within the + same academic year and subject, ordered by administration_window. diff --git a/src/dbt/kipptaf/models/google/sheets/sources-external.yml b/src/dbt/kipptaf/models/google/sheets/sources-external.yml index 590f7974cd..7e9f93328f 100644 --- a/src/dbt/kipptaf/models/google/sheets/sources-external.yml +++ b/src/dbt/kipptaf/models/google/sheets/sources-external.yml @@ -67,13 +67,6 @@ sources: - sheets - student_graduation_path_cutoffs - name: src_google_sheets__state_test_comparison - external: - options: - format: GOOGLE_SHEETS - uris: - - https://docs.google.com/spreadsheets/d/1yS6xU7ygiOrrtc29pUc3jr590qk7ttag3RuzVHaPOv8 - sheet_range: src_google_sheets__state_test_comparison - skip_leading_rows: 1 config: meta: dagster: @@ -82,13 +75,20 @@ sources: - google - sheets - state_test_comparison + external: + options: + format: GOOGLE_SHEETS + uris: + - https://docs.google.com/spreadsheets/d/1yS6xU7ygiOrrtc29pUc3jr590qk7ttag3RuzVHaPOv8 + sheet_range: src_google_sheets__state_test_comparison + skip_leading_rows: 1 - name: src_google_sheets__state_test_comparison_demographics external: options: format: GOOGLE_SHEETS uris: - https://docs.google.com/spreadsheets/d/1yS6xU7ygiOrrtc29pUc3jr590qk7ttag3RuzVHaPOv8 - sheet_range: src_google_sheets__state_test_comparison_demographics + sheet_range: src_google_sheets__state_test_comparison_demographics_v2 skip_leading_rows: 1 config: meta: diff --git a/src/dbt/kipptaf/models/google/sheets/staging/properties/stg_google_sheets__state_test_comparison_demographics.yml b/src/dbt/kipptaf/models/google/sheets/staging/properties/stg_google_sheets__state_test_comparison_demographics.yml index a6f5776e06..f5a85ac079 100644 --- a/src/dbt/kipptaf/models/google/sheets/staging/properties/stg_google_sheets__state_test_comparison_demographics.yml +++ b/src/dbt/kipptaf/models/google/sheets/staging/properties/stg_google_sheets__state_test_comparison_demographics.yml @@ -1,21 +1,113 @@ models: - name: stg_google_sheets__state_test_comparison_demographics + description: > + Reference sheet for state assessment comparisons to external entities + (City LEA, State, Neighborhood Schools) broken out by demographic group, + as available from official sources. ALG01 has a special case for school + level — overall comparisons by entity are split by HS and MS, but + demographic groupings combine MS and HS (assessment grades 8+) because + official sources do not provide demographic breakdowns by school level. + ALG02 totals are 10th-grade only; demographic group totals include all + students who took ALG02 regardless of grade. GEO01 follows the same + pattern for grades 9 and 10. + data_tests: + - dbt_utils.unique_combination_of_columns: + arguments: + combination_of_columns: + - academic_year + - aligned_test_code + - school_level + - region + - comparison_entity + - comparison_demographic_group + - comparison_demographic_subgroup columns: - - name: Academic_Year + - name: academic_year data_type: int64 - - name: Test_Name + description: Academic year the benchmarks apply to. + + - name: assessment_name data_type: string - - name: Test_Code + description: State assessment test name (e.g. NJSLA, NJGPA). + + - name: season data_type: string - - name: Region + description: Testing season (always Spring for benchmarks). + + - name: school_level data_type: string - - name: Comparison_Entity + description: > + School level the benchmark applies to (ES, MS, HS, or hybrid MS_HS for + ALG01, or HS_09 and HS_10 for ALG02, GEO01). + + - name: grade_range_band data_type: string - - name: Comparison_Demographic_Group + description: Grade band for the benchmark (e.g. 3-8, HS). + + - name: discipline data_type: string - - name: Comparison_Demographic_Subgroup + description: Subject discipline (Math, ELA, Science, Social Studies). + + - name: aligned_test_code data_type: string - - name: Percent_Proficient + description: > + Test code normalized for stacked reporting across assessment programs + — science codes SC05, SC08, SC11 are remapped to SCI05, SCI08, SCI11 + to match the standard codes used across all sources; all other codes + pass through unchanged. + + - name: region + data_type: string + description: > + Network region the comparison applies to (Newark, Camden, Miami, + Paterson). + + - name: comparison_entity + data_type: string + description: > + External entity being compared to (City, State, Neighborhood Schools). + + - name: comparison_demographic_group + data_type: string + description: > + Raw demographic group label from the source sheet (e.g. Grade, + Race/Ethnicity, Gender). + + - name: comparison_demographic_subgroup + data_type: string + description: > + Raw demographic subgroup label from the source sheet (e.g. Grade - 05, + Black or African American, Female). + + - name: percent_proficient data_type: float64 - - name: Total_Students + description: Percent of students scoring proficient or above. + + - name: total_students data_type: int64 + description: Total number of students tested. + + - name: aligned_level_test_code + data_type: string + description: > + Test code adjusted for school level — ALG01 rows with a school_level + other than MS_HS are suffixed with the school level (e.g. 'ALG01_HS'); + all other test codes are passed through unchanged. + + - name: aligned_comparison_demographic_group + data_type: string + description: > + Demographic group after alignment — Grade rows with subgroups 08/09/10 + are remapped to 'Total'. + + - name: aligned_comparison_demographic_subgroup + data_type: string + description: > + Demographic subgroup after alignment — Grade group rows are remapped + to 'All Students'. + + - name: total_proficient_students + data_type: float64 + description: > + Derived count of proficient students (percent_proficient * + total_students, rounded to 0 decimal places). diff --git a/src/dbt/kipptaf/models/google/sheets/staging/stg_google_sheets__state_test_comparison_demographics.sql b/src/dbt/kipptaf/models/google/sheets/staging/stg_google_sheets__state_test_comparison_demographics.sql index b43f3385a3..b70dc470d4 100644 --- a/src/dbt/kipptaf/models/google/sheets/staging/stg_google_sheets__state_test_comparison_demographics.sql +++ b/src/dbt/kipptaf/models/google/sheets/staging/stg_google_sheets__state_test_comparison_demographics.sql @@ -1,7 +1,87 @@ -select *, -from - {{ - source( - "google_sheets", "src_google_sheets__state_test_comparison_demographics" - ) - }} +with + custom_rows as ( + select + academic_year, + assessment_name, + season, + school_level, + grade_range_band, + discipline, + aligned_test_code, + region, + comparison_entity, + comparison_demographic_group, + comparison_demographic_subgroup, + percent_proficient, + total_students, + + from + {{ + source( + "google_sheets", + "src_google_sheets__state_test_comparison_demographics", + ) + }} + + union all + + -- HS-only ALG01 totals are not provided by official comp data sources + select + academic_year, + assessment_name, + season, + grade_range_band as school_level, + grade_range_band, + discipline, + aligned_test_code, + region, + comparison_entity, + 'Total' as comparison_demographic_group, + 'All Students' as comparison_demographic_subgroup, + + sum(percent_proficient) as percent_proficient, + sum(total_students) as total_students, + + from + {{ + source( + "google_sheets", + "src_google_sheets__state_test_comparison_demographics", + ) + }} + where school_level in ('HS_09', 'HS_10') and aligned_test_code = 'ALG01' + group by + academic_year, + assessment_name, + season, + grade_range_band, + discipline, + aligned_test_code, + region, + comparison_entity + ) + +select + *, + + case + when aligned_test_code = 'ALG01' and school_level != 'MS_HS' + then concat(aligned_test_code, '_', school_level) + else aligned_test_code + end as aligned_level_test_code, + + if( + comparison_demographic_subgroup in ('Grade - 08', 'Grade - 09', 'Grade - 10'), + 'Total', + comparison_demographic_group + ) as aligned_comparison_demographic_group, + + if( + comparison_demographic_group = 'Grade', + 'All Students', + comparison_demographic_subgroup + ) as aligned_comparison_demographic_subgroup, + + round(percent_proficient * total_students, 0) as total_proficient_students, + +from custom_rows diff --git a/src/dbt/kipptaf/models/iready/intermediate/int_iready__diagnostic_results.sql b/src/dbt/kipptaf/models/iready/intermediate/int_iready__diagnostic_results.sql index 28214543ef..4f7cdb77d3 100644 --- a/src/dbt/kipptaf/models/iready/intermediate/int_iready__diagnostic_results.sql +++ b/src/dbt/kipptaf/models/iready/intermediate/int_iready__diagnostic_results.sql @@ -125,6 +125,21 @@ select right(rt.code, 1) as round_number, + case + when wc.overall_relative_placement_int <= 2 + then 'Below/Far Below' + when wc.overall_relative_placement_int = 3 + then 'Approaching' + when wc.overall_relative_placement_int >= 4 + then 'At/Above' + end as iready_proficiency, + + if( + cwp.scale_low - wc.most_recent_overall_scale_score <= 0, + 0, + cwp.scale_low - wc.most_recent_overall_scale_score + ) as scale_points_to_proficiency, + round( wc.most_recent_diagnostic_gain / wc.annual_typical_growth_measure, 2 ) as progress_to_typical, @@ -133,12 +148,6 @@ select wc.most_recent_diagnostic_gain / wc.annual_stretch_growth_measure, 2 ) as progress_to_stretch, - if( - cwp.scale_low - wc.most_recent_overall_scale_score <= 0, - 0, - cwp.scale_low - wc.most_recent_overall_scale_score - ) as scale_points_to_proficiency, - row_number() over ( partition by wc._dbt_source_relation, @@ -148,6 +157,7 @@ select rt.name order by wc.completion_date desc ) as rn_subj_round, + from window_calcs as wc left join {{ ref("stg_google_sheets__reporting__terms") }} as rt diff --git a/src/dbt/kipptaf/models/iready/intermediate/properties/int_iready__diagnostic_results.yml b/src/dbt/kipptaf/models/iready/intermediate/properties/int_iready__diagnostic_results.yml index 9cb09833a5..faf1769f4e 100644 --- a/src/dbt/kipptaf/models/iready/intermediate/properties/int_iready__diagnostic_results.yml +++ b/src/dbt/kipptaf/models/iready/intermediate/properties/int_iready__diagnostic_results.yml @@ -280,6 +280,8 @@ models: data_type: int64 - name: round_number data_type: string + - name: iready_proficiency + data_type: string - name: scale_points_to_proficiency data_type: int64 - name: sublevel_number_with_typical diff --git a/src/dbt/kipptaf/models/marts/dim_state_assessment_benchmarks.sql b/src/dbt/kipptaf/models/marts/dim_state_assessment_benchmarks.sql index 52662cfe94..cc00cb509d 100644 --- a/src/dbt/kipptaf/models/marts/dim_state_assessment_benchmarks.sql +++ b/src/dbt/kipptaf/models/marts/dim_state_assessment_benchmarks.sql @@ -1,8 +1,14 @@ select academic_year, - test_name, - test_code, + assessment_name, + season, + school_level, + grade_range_band, + discipline, + aligned_test_code, region, + aligned_comparison_demographic_group, + aligned_comparison_demographic_subgroup, max( case when comparison_entity = 'City' then percent_proficient end @@ -24,11 +30,44 @@ select case when comparison_entity = 'Neighborhood Schools' then total_students end ) as neighborhood_schools_total_students, + max( + case when comparison_entity = 'City' then total_proficient_students end + ) as city_total_proficient_students, + max( + case when comparison_entity = 'State' then total_proficient_students end + ) as state_total_proficient_students, + max( + case + when comparison_entity = 'Neighborhood Schools' + then total_proficient_students + end + ) as neighborhood_schools_total_proficient_students, + {{ dbt_utils.generate_surrogate_key( - ["academic_year", "test_name", "test_code", "region"] + [ + "academic_year", + "assessment_name", + "aligned_test_code", + "region", + "school_level", + "grade_range_band", + "season", + "aligned_comparison_demographic_group", + "aligned_comparison_demographic_subgroup", + ] ) }} as state_assessment_benchmarks_key, -from {{ ref("stg_google_sheets__state_test_comparison") }} -group by academic_year, test_name, test_code, region +from {{ ref("stg_google_sheets__state_test_comparison_demographics") }} +group by + academic_year, + assessment_name, + season, + school_level, + grade_range_band, + discipline, + aligned_test_code, + region, + aligned_comparison_demographic_group, + aligned_comparison_demographic_subgroup diff --git a/src/dbt/kipptaf/models/marts/properties/dim_state_assessment_benchmarks.yml b/src/dbt/kipptaf/models/marts/properties/dim_state_assessment_benchmarks.yml index 81a6c8cdb3..303f9d25ca 100644 --- a/src/dbt/kipptaf/models/marts/properties/dim_state_assessment_benchmarks.yml +++ b/src/dbt/kipptaf/models/marts/properties/dim_state_assessment_benchmarks.yml @@ -4,8 +4,11 @@ models: columns: - name: state_assessment_benchmarks_key data_type: string - description: - Surrogate key on (academic_year, test_name, test_code, region). + description: > + Surrogate key on (academic_year, assessment_name, aligned_test_code, + region, school_level, grade_range_band, season, + aligned_comparison_demographic_group, + aligned_comparison_demographic_subgroup). data_tests: - unique @@ -13,20 +16,50 @@ models: data_type: int64 description: Academic year the benchmarks apply to. - - name: test_name + - name: assessment_name data_type: string description: State assessment test name (e.g. NJSLA, NJGPA). - - name: test_code + - name: season data_type: string - description: State assessment test code (e.g. ELA05, ALG1). + description: Testing season (always Spring for benchmarks). - - name: region + - name: school_level data_type: string description: + School level the benchmark applies to (ES, MS, HS, or hybrid MS_HS for + ALG01, or HS_09, and HS_10 for ALG02, GEO01). + + - name: grade_range_band + data_type: string + description: Grade band for the benchmark (e.g. 3-8, HS). + + - name: discipline + data_type: string + description: Subject discipline (Math, ELA, Science, Social Studies). + + - name: aligned_test_code + data_type: string + description: State assessment test code (e.g. ELA05, ALG01). + + - name: region + data_type: string + description: > Network region the comparison applies to (Newark, Camden, Miami, Paterson). + - name: aligned_comparison_demographic_group + data_type: string + description: > + Demographic group after alignment — Grade rows with subgroups 08/09/10 + are remapped to 'Total'. + + - name: aligned_comparison_demographic_subgroup + data_type: string + description: > + Demographic subgroup after alignment — Grade group rows are remapped + to 'All Students'. + - name: city_percent_proficient data_type: float64 description: City-wide percent proficient on this assessment. @@ -37,7 +70,9 @@ models: - name: neighborhood_schools_percent_proficient data_type: float64 - description: Neighborhood schools percent proficient on this assessment. + description: + Neighborhood schools percent proficient on this assessment (applies to + FL only). - name: city_total_students data_type: int64 @@ -49,4 +84,19 @@ models: - name: neighborhood_schools_total_students data_type: int64 - description: Total students tested at neighborhood schools. + description: + Total students tested at neighborhood schools (applies to FL only). + + - name: city_total_proficient_students + data_type: float64 + description: Total proficient students city-wide. + + - name: state_total_proficient_students + data_type: float64 + description: Total proficient students state-wide. + + - name: neighborhood_schools_total_proficient_students + data_type: float64 + description: + Total proficient students at neighborhood schools (applies to FL + only). diff --git a/src/dbt/kipptaf/models/pearson/intermediate/int_pearson__all_assessments.sql b/src/dbt/kipptaf/models/pearson/intermediate/int_pearson__all_assessments.sql index 5ca1b53944..bdeaea42ff 100644 --- a/src/dbt/kipptaf/models/pearson/intermediate/int_pearson__all_assessments.sql +++ b/src/dbt/kipptaf/models/pearson/intermediate/int_pearson__all_assessments.sql @@ -42,63 +42,144 @@ with ], ) }} + ), + + transformations as ( + select + u._dbt_source_relation, + u.academic_year, + u.americanindianoralaskanative, + u.asian, + u.assessment_name, + u.assessmentgrade, + u.assessmentyear, + u.blackorafricanamerican, + u.discipline, + u.hispanicorlatinoethnicity, + u.is_proficient, + u.is_bl_fb, + u.nativehawaiianorotherpacificislander, + u.period, + u.firstname, + u.lastorsurname, + u.`subject`, + u.testcode, + u.studenttestuuid, + u.test_grade, + u.testperformancelevel_text, + u.testperformancelevel, + u.testscalescore, + u.twoormoreraces, + u.white, + + 'Actual' as results_type, + 'KTAF NJ' as district_state, + + cast(u.statestudentidentifier as string) as statestudentidentifier, + + coalesce(u.studentwithdisabilities in ('504', 'B'), false) as is_504, + + coalesce( + x.student_number, u.localstudentidentifier + ) as localstudentidentifier, + + case + u.testcode + when 'SC05' + then 'SCI05' + when 'SC08' + then 'SCI08' + when 'SC11' + then 'SCI11' + else u.testcode + end as aligned_test_code, + + case + when u.twoormoreraces = 'Y' + then 'T' + when u.hispanicorlatinoethnicity = 'Y' + then 'H' + when u.americanindianoralaskanative = 'Y' + then 'I' + when u.asian = 'Y' + then 'A' + when u.blackorafricanamerican = 'Y' + then 'B' + when u.nativehawaiianorotherpacificislander = 'Y' + then 'P' + when u.white = 'Y' + then 'W' + end as race_ethnicity, + + case + when u.`subject` like 'English Language Arts%' + then 'Text Study' + when u.`subject` in ('Algebra I', 'Algebra II', 'Geometry') + then 'Mathematics' + else u.`subject` + end as illuminate_subject, + + case + when u.assessment_name = 'NJSLA' and u.testperformancelevel <= 2 + then 'Below/Far Below' + when u.assessment_name = 'NJSLA' and u.testperformancelevel = 3 + then 'Approaching' + when u.assessment_name = 'NJSLA' and u.testperformancelevel >= 4 + then 'At/Above' + end as njsla_aggregated_proficiency, + + if(u.englishlearnerel = 'Y', true, false) as lep_status, + + if( + u.studentwithdisabilities in ('IEP', 'B'), 'Has IEP', 'No IEP' + ) as iep_status, + + if(u.`period` = 'FallBlock', 'Fall', u.`period`) as `admin`, + + if(u.`period` = 'FallBlock', 'Fall', u.`period`) as season, + + if( + u.`subject` = 'English Language Arts/Literacy', + 'English Language Arts', + u.`subject` + ) as aligned_subject, + + if(u.is_proficient, 1, 0) as is_proficient_int, + + from union_relations as u + left join + {{ ref("stg_google_sheets__pearson__student_crosswalk") }} as x + on u.studenttestuuid = x.student_test_uuid ) select - u._dbt_source_relation, - u.academic_year, - u.americanindianoralaskanative, - u.asian, - u.assessment_name, - u.assessmentgrade, - u.assessmentyear, - u.blackorafricanamerican, - u.discipline, - u.hispanicorlatinoethnicity, - u.is_proficient, - u.is_bl_fb, - u.nativehawaiianorotherpacificislander, - u.period, - u.firstname, - u.lastorsurname, - u.subject, - u.testcode, - u.studenttestuuid, - u.test_grade, - u.testperformancelevel_text, - u.testperformancelevel, - u.testscalescore, - u.twoormoreraces, - u.white, - - cast(u.statestudentidentifier as string) as statestudentidentifier, - - coalesce(u.studentwithdisabilities in ('504', 'B'), false) as is_504, - - coalesce(x.student_number, u.localstudentidentifier) as localstudentidentifier, - - if(u.englishlearnerel = 'Y', true, false) as lep_status, - - if(u.studentwithdisabilities in ('IEP', 'B'), 'Has IEP', 'No IEP') as iep_status, + *, case - when u.twoormoreraces = 'Y' - then 'T' - when u.hispanicorlatinoethnicity = 'Y' - then 'H' - when u.americanindianoralaskanative = 'Y' - then 'I' - when u.asian = 'Y' - then 'A' - when u.blackorafricanamerican = 'Y' - then 'B' - when u.nativehawaiianorotherpacificislander = 'Y' - then 'P' - when u.white = 'Y' - then 'W' - end as race_ethnicity, - -from union_relations as u -left join - {{ ref("stg_google_sheets__pearson__student_crosswalk") }} as x - on u.studenttestuuid = x.student_test_uuid + when race_ethnicity = 'B' + then 'African American' + when race_ethnicity = 'A' + then 'Asian' + when race_ethnicity = 'I' + then 'American Indian' + when race_ethnicity = 'H' + then 'Hispanic' + when race_ethnicity = 'P' + then 'Native Hawaiian' + when race_ethnicity = 'T' + then 'Other' + when race_ethnicity = 'W' + then 'White' + when race_ethnicity is null + then 'Blank' + end as aligned_aggregate_ethnicity, + + if(lep_status, 'ML', 'Not ML') as aligned_ml_status, + + if( + iep_status = 'Has IEP', + 'Students With Disabilities', + 'Students Without Disabilities' + ) as aligned_iep_status, + +from transformations diff --git a/src/dbt/kipptaf/models/pearson/intermediate/int_pearson__student_list_report.sql b/src/dbt/kipptaf/models/pearson/intermediate/int_pearson__student_list_report.sql new file mode 100644 index 0000000000..65e5954b80 --- /dev/null +++ b/src/dbt/kipptaf/models/pearson/intermediate/int_pearson__student_list_report.sql @@ -0,0 +1,89 @@ +with + scores as ( + select + _dbt_source_relation, + academic_year, + state_student_identifier, + local_student_identifier, + last_or_surname, + first_name, + date_of_birth, + test_type, + scale_score, + performance_level, + administration, + + 'Preliminary' as results_type, + 'KTAF NJ' as district_state, + + cast(state_student_identifier as string) as state_id, + + case + when test_name like '%Mathematics%' + then 'Math' + when test_name in ('Algebra I', 'Geometry') + then 'Math' + else 'ELA' + end as discipline, + + case + when test_name like '%Mathematics%' + then 'Mathematics' + when test_name in ('Algebra I', 'Geometry') + then 'Mathematics' + else 'English Language Arts' + end as `subject`, + + case + when test_name = 'ELA Graduation Proficiency' + then 'ELAGP' + when test_name = 'Mathematics Graduation Proficiency' + then 'MATGP' + when test_name = 'Geometry' + then 'GEO01' + when test_name = 'Algebra I' + then 'ALG01' + when test_name like '%Mathematics%' + then concat('MAT', regexp_extract(test_name, r'.{6}(.{2})')) + when test_name like '%ELA%' + then concat('ELA', regexp_extract(test_name, r'.{6}(.{2})')) + end as aligned_test_code, + + case + when performance_level = 'Did Not Yet Meet Expectations' + then 1 + when performance_level = 'Partially Met Expectations' + then 2 + when performance_level = 'Approached Expectations' + then 3 + when performance_level = 'Met Expectations' + then 4 + when performance_level = 'Exceeded Expectations' + then 5 + when performance_level = 'Not Yet Graduation Ready' + then 1 + when performance_level = 'Graduation Ready' + then 2 + end as performance_band_level, + + from {{ ref("stg_pearson__student_list_report") }} + ) + +select + *, + + if( + performance_level + in ('Met Expectations', 'Exceeded Expectations', 'Graduation Ready'), + true, + false + ) as is_proficient, + + if( + performance_level + in ('Met Expectations', 'Exceeded Expectations', 'Graduation Ready'), + 1, + 0 + ) as is_proficient_int, + +from scores diff --git a/src/dbt/kipptaf/models/pearson/intermediate/properties/int_pearson__all_assessments.yml b/src/dbt/kipptaf/models/pearson/intermediate/properties/int_pearson__all_assessments.yml index 2320ebb6f0..67d1965826 100644 --- a/src/dbt/kipptaf/models/pearson/intermediate/properties/int_pearson__all_assessments.yml +++ b/src/dbt/kipptaf/models/pearson/intermediate/properties/int_pearson__all_assessments.yml @@ -1,65 +1,239 @@ models: - name: int_pearson__all_assessments + description: > + Network-level union of Pearson state assessment results (PARCC, NJSLA, + NJSLA Science, NJGPA) enriched with student crosswalk lookups and + standardized metadata. Computes aligned demographic labels, proficiency + banding, and subject/test-code normalization for use in reporting and + demographic comparison models. + data_tests: + - dbt_utils.unique_combination_of_columns: + arguments: + combination_of_columns: + - studenttestuuid columns: - name: _dbt_source_relation data_type: string + description: Source relation identifier from dbt_utils.union_relations. + + - name: academic_year + data_type: int64 + description: Academic year the assessment was administered. + - name: americanindianoralaskanative data_type: string + description: > + American Indian or Alaska Native race indicator from Pearson ('Y' or + null). + - name: asian data_type: string + description: Asian race indicator from Pearson ('Y' or null). + + - name: assessment_name + data_type: string + description: Assessment program name (e.g. NJSLA, NJGPA, PARCC). + - name: assessmentgrade data_type: string + description: Assessment grade level as reported by Pearson. + - name: assessmentyear data_type: string + description: Assessment year as reported by Pearson (string format). + - name: blackorafricanamerican data_type: string - - name: englishlearnerel + description: + Black or African American race indicator from Pearson ('Y' or null). + + - name: discipline data_type: string + description: Subject discipline category (Math, ELA, Science, etc.). + - name: hispanicorlatinoethnicity data_type: string - - name: localstudentidentifier - data_type: int64 - - name: firstname - data_type: string - - name: lastorsurname - data_type: string + description: + Hispanic or Latino ethnicity indicator from Pearson ('Y' or null). + + - name: is_proficient + data_type: boolean + description: + Whether the student scored at or above the proficiency threshold. + + - name: is_bl_fb + data_type: boolean + description: > + Whether the student scored at the Below or Far Below proficiency band. + - name: nativehawaiianorotherpacificislander data_type: string + description: > + Native Hawaiian or Other Pacific Islander race indicator from Pearson + ('Y' or null). + - name: period data_type: string - - name: studentwithdisabilities + description: > + Testing period from Pearson (e.g. Spring, FallBlock); used to derive + admin and season. Spring is the only period officially reported. + + - name: firstname + data_type: string + description: Student first name as reported by Pearson. + + - name: lastorsurname data_type: string + description: Student last name or surname as reported by Pearson. + - name: subject data_type: string + quote: true + description: + Full subject name as reported by Pearson (e.g. English Language + Arts/Literacy, Mathematics). + - name: testcode data_type: string + description: Raw test code from Pearson (e.g. ELA05, SC08, ALG01). + + - name: studenttestuuid + data_type: string + description: + Globally unique identifier for the student's test-taking event. + + - name: test_grade + data_type: int64 + description: Numeric grade level at which the test was taken. + + - name: testperformancelevel_text + data_type: string + description: + Performance level label as reported by Pearson (e.g. 'Level 4'). + - name: testperformancelevel data_type: float64 + description: Numeric performance level as reported by Pearson. + - name: testscalescore data_type: float64 + description: Student's raw scale score on the assessment. + - name: twoormoreraces data_type: string + description: Two or more races indicator from Pearson ('Y' or null). + - name: white data_type: string - - name: assessment_name - data_type: string - - name: academic_year - data_type: int64 - - name: test_grade - data_type: int64 - - name: discipline + description: White race indicator from Pearson ('Y' or null). + + - name: results_type data_type: string - - name: is_proficient - data_type: boolean - - name: testperformancelevel_text + description: + Always 'Actual' for Pearson results (no preliminary scores). + + - name: district_state data_type: string + description: > + Network district/state identifier; always 'KTAF NJ' for this model. + - name: statestudentidentifier data_type: string - - name: is_504 - data_type: boolean + description: State student identifier cast to string. + + - name: localstudentidentifier + data_type: int64 + description: > + Local student identifier — resolved from the Pearson student crosswalk + when available, otherwise the raw value from the source file. Used to + work around incorrect studentidentifiers provided during testing. + + - name: aligned_test_code + data_type: string + description: > + Test code normalized for stacked reporting across assessment programs + — science codes SC05, SC08, SC11 are remapped to SCI05, SCI08, SCI11 + to match the standard codes used across all sources; all other codes + pass through unchanged. + + - name: race_ethnicity + data_type: string + description: > + Single-character race/ethnicity code derived from Pearson indicator + flags using a priority hierarchy (T, H, I, A, B, P, W). + + - name: illuminate_subject + data_type: string + description: > + Standardized subject label for Illuminate/reporting — ELA subjects map + to 'Text Study'; Algebra I, Algebra II, and Geometry map to + 'Mathematics'; all others pass through unchanged. + + - name: njsla_aggregated_proficiency + data_type: string + description: > + Aggregated proficiency band for NJSLA: levels 1-2 = 'Below/Far Below', + level 3 = 'Approaching', level 4+ = 'At/Above'; null for non-NJSLA + assessments. + - name: lep_status data_type: boolean + description: > + Whether the student is classified as an English Learner/EL, derived + from the englishlearnerel indicator. + - name: iep_status data_type: string - - name: race_ethnicity - data_type: string + description: > + IEP status label — 'Has IEP' when studentwithdisabilities is 'IEP' or + 'B', otherwise 'No IEP'. + + - name: admin + data_type: string + quote: true + description: > + Testing administration period with 'FallBlock' normalized to 'Fall'; + reserved word alias for Tableau. + + - name: season + data_type: string + description: > + Testing season with 'FallBlock' normalized to 'Fall'; same logic as + admin. + + - name: aligned_subject + data_type: string + description: > + Subject normalized for stacked reporting across assessment programs — + 'English Language Arts/Literacy' is shortened to 'English Language + Arts' to match the label used in other sources; all other subjects + pass through unchanged. + + - name: is_proficient_int + data_type: int64 + description: + Integer representation of is_proficient (1 = true, 0 = false). + + - name: aligned_aggregate_ethnicity + data_type: string + description: > + Race/ethnicity label normalized for stacked reporting across + assessment programs — expands the single-character race_ethnicity code + to a full label (e.g. B = 'African American', H = 'Hispanic') that + matches the demographic labels used in comparison sources; null maps + to 'Blank'. + + - name: aligned_ml_status + data_type: string + description: > + Multilingual learner status normalized for stacked reporting across + assessment programs — 'ML' when lep_status is true, 'Not ML' + otherwise, using the label convention shared across comparison + sources. + + - name: aligned_iep_status + data_type: string + description: > + IEP status normalized for stacked reporting across assessment programs + — 'Students With Disabilities' or 'Students Without Disabilities', + using the label convention shared across comparison sources. diff --git a/src/dbt/kipptaf/models/pearson/intermediate/properties/int_pearson__student_list_report.yml b/src/dbt/kipptaf/models/pearson/intermediate/properties/int_pearson__student_list_report.yml new file mode 100644 index 0000000000..bf35b7966a --- /dev/null +++ b/src/dbt/kipptaf/models/pearson/intermediate/properties/int_pearson__student_list_report.yml @@ -0,0 +1,124 @@ +models: + - name: int_pearson__student_list_report + description: > + Intermediate transformation of preliminary Pearson student list report + data, unioned across Newark, Camden, and Paterson regions. Derives + discipline, subject, aligned test code, and performance band level from + the raw performance_level and test_name columns, and computes + is_proficient and is_proficient_int for use in demographic comparison + models. + data_tests: + - dbt_utils.unique_combination_of_columns: + arguments: + combination_of_columns: + - _dbt_source_relation + - academic_year + - administration + - state_id + - aligned_test_code + columns: + - name: _dbt_source_relation + data_type: string + description: Source relation identifier from dbt_utils.union_relations. + + - name: academic_year + data_type: int64 + description: Academic year the assessment was administered. + + - name: state_student_identifier + data_type: int64 + description: State student identifier from the source file. + + - name: local_student_identifier + data_type: int64 + description: Local student identifier from the source file. + + - name: last_or_surname + data_type: string + description: Student last name or surname as reported by Pearson. + + - name: first_name + data_type: string + description: Student first name as reported by Pearson. + + - name: date_of_birth + data_type: string + description: Student date of birth as reported by Pearson. + + - name: test_type + data_type: string + description: Assessment test type from the Dagster partition key. + + - name: scale_score + data_type: int64 + description: Student's raw scale score on the assessment. + + - name: performance_level + data_type: string + description: > + Performance level label as reported by Pearson (e.g. 'Met + Expectations', 'Graduation Ready'). + + - name: administration + data_type: string + description: + Assessment administration period from the Dagster partition key. + + - name: results_type + data_type: string + description: + Always 'Preliminary' — student list reports contain pre-release + scores. + + - name: district_state + data_type: string + description: > + Network district/state identifier; always 'KTAF NJ' for this model. + + - name: state_id + data_type: string + description: State student identifier cast to string. + + - name: discipline + data_type: string + description: > + Subject discipline derived from test_name — + Mathematics/Algebra/Geometry map to 'Math', all others map to 'ELA'. + + - name: subject + data_type: string + quote: true + description: > + Subject normalized for stacked reporting across assessment programs — + Mathematics/Algebra/Geometry map to 'Mathematics', all others map to + 'English Language Arts', matching the label convention used in + comparison sources. + + - name: aligned_test_code + data_type: string + description: > + Test code normalized for stacked reporting across assessment programs + — derived from test_name: graduation proficiency tests map to + ELAGP/MATGP, Geometry to GEO01, Algebra I to ALG01; grade-level math + and ELA tests extract the grade from the name and prefix with MAT/ELA. + + - name: performance_band_level + data_type: int64 + description: > + Numeric performance band level derived from performance_level: 'Did + Not Yet Meet Expectations'/'Not Yet Graduation Ready' = 1, 'Partially + Met Expectations' = 2, 'Approached Expectations' = 3, 'Met + Expectations' = 4, 'Exceeded Expectations' = 5, 'Graduation Ready' = + 2. + + - name: is_proficient + data_type: boolean + description: > + Whether the student scored proficient or above — true when + performance_level is 'Met Expectations', 'Exceeded Expectations', or + 'Graduation Ready'. + + - name: is_proficient_int + data_type: int64 + description: + Integer representation of is_proficient (1 = true, 0 = false). diff --git a/src/dbt/kipptaf/models/powerschool/base/base_powerschool__course_enrollments.sql b/src/dbt/kipptaf/models/powerschool/base/base_powerschool__course_enrollments.sql index 944ef84c66..20d75b3cfd 100644 --- a/src/dbt/kipptaf/models/powerschool/base/base_powerschool__course_enrollments.sql +++ b/src/dbt/kipptaf/models/powerschool/base/base_powerschool__course_enrollments.sql @@ -67,6 +67,8 @@ select else ur.courses_credittype end as courses_credittype, + if(csc.discipline = 'SOC', 'Civics', csc.discipline) as standardized_discipline, + row_number() over ( partition by ur._dbt_source_relation, ur.cc_studyear, csc.illuminate_subject_area diff --git a/src/dbt/kipptaf/models/powerschool/base/properties/base_powerschool__course_enrollments.yml b/src/dbt/kipptaf/models/powerschool/base/properties/base_powerschool__course_enrollments.yml index 7704d4816a..e9e3a06a1f 100644 --- a/src/dbt/kipptaf/models/powerschool/base/properties/base_powerschool__course_enrollments.yml +++ b/src/dbt/kipptaf/models/powerschool/base/properties/base_powerschool__course_enrollments.yml @@ -483,5 +483,7 @@ models: → SCI, Homeroom → HR) caused by a 2023 PowerSchool plugin glitch in one region that produced non-standard values for that year's records. data_type: string + - name: standardized_discipline + data_type: string - name: rn_student_year_illuminate_subject_desc data_type: int64 diff --git a/src/dbt/kipptaf/models/students/intermediate/int_extracts__student_enrollments.sql b/src/dbt/kipptaf/models/students/intermediate/int_extracts__student_enrollments.sql index b4661edb66..034f8f8a9f 100644 --- a/src/dbt/kipptaf/models/students/intermediate/int_extracts__student_enrollments.sql +++ b/src/dbt/kipptaf/models/students/intermediate/int_extracts__student_enrollments.sql @@ -289,6 +289,10 @@ select false ) as student_slideback, + case + e.gender when 'F' then 'Female' when 'M' then 'Male' when 'X' then 'Non-Binary' + end as aligned_gender, + case e.enroll_status when -2 diff --git a/src/dbt/kipptaf/models/students/intermediate/int_extracts__student_enrollments_courses.sql b/src/dbt/kipptaf/models/students/intermediate/int_extracts__student_enrollments_courses.sql new file mode 100644 index 0000000000..74b19fa8b8 --- /dev/null +++ b/src/dbt/kipptaf/models/students/intermediate/int_extracts__student_enrollments_courses.sql @@ -0,0 +1,32 @@ +select + e.* except ( + lastfirst, + last_name, + first_name, + middle_name, + school_abbreviation, + advisory_section_number, + student_email_google, + salesforce_contact_id, + salesforce_contact_df_has_fafsa, + salesforce_contact_college_match_display_gpa, + salesforce_contact_college_match_gpa_band, + salesforce_contact_owner_name, + state_studentnumber, + `state` + ), + + e.lastfirst as student_name, + e.last_name as student_last_name, + e.first_name as student_first_name, + e.middle_name as student_middle_name, + e.school_abbreviation as school, + e.advisory_section_number as team, + e.student_email_google as student_email, + e.salesforce_contact_id as salesforce_id, + e.salesforce_contact_df_has_fafsa as has_fafsa, + e.salesforce_contact_college_match_display_gpa as college_match_gpa, + e.salesforce_contact_college_match_gpa_band as college_match_gpa_bands, + e.salesforce_contact_owner_name as contact_owner_name, + +from {{ ref("base_powerschool__student_enrollments") }} as e diff --git a/src/dbt/kipptaf/models/students/intermediate/int_extracts__student_enrollments_subjects.sql b/src/dbt/kipptaf/models/students/intermediate/int_extracts__student_enrollments_subjects.sql index 213bb31432..5fc5c39a99 100644 --- a/src/dbt/kipptaf/models/students/intermediate/int_extracts__student_enrollments_subjects.sql +++ b/src/dbt/kipptaf/models/students/intermediate/int_extracts__student_enrollments_subjects.sql @@ -54,27 +54,13 @@ with localstudentidentifier, is_proficient, + illuminate_subject as `subject`, + njsla_aggregated_proficiency as njsla_proficiency, + academic_year + 1 as academic_year_plus, cast(statestudentidentifier as string) as statestudentidentifier, - case - when `subject` like 'English Language Arts%' - then 'Text Study' - when `subject` in ('Algebra I', 'Algebra II', 'Geometry') - then 'Mathematics' - else `subject` - end as `subject`, - - case - when testperformancelevel <= 2 - then 'Below/Far Below' - when testperformancelevel = 3 - then 'Approaching' - when testperformancelevel >= 4 - then 'At/Above' - end as njsla_proficiency, - from {{ ref("int_pearson__all_assessments") }} union all @@ -86,27 +72,13 @@ with is_proficient, + illuminate_subject as `subject`, + fast_aggregated_proficiency as proficiency, + academic_year + 1 as academic_year_plus, student_id as statestudentidentifier, - case - when assessment_subject like 'English Language Arts%' - then 'Text Study' - when assessment_subject in ('Algebra I', 'Algebra II', 'Geometry') - then 'Mathematics' - else assessment_subject - end as `subject`, - - case - when achievement_level_int = 1 - then 'Below/Far Below' - when achievement_level_int = 2 - then 'Approaching' - when achievement_level_int >= 3 - then 'At/Above' - end as proficiency, - from {{ ref("int_fldoe__all_assessments") }} where scale_score is not null @@ -118,18 +90,10 @@ with select student_id, `subject`, + iready_proficiency, academic_year_int + 1 as academic_year_plus, - case - when overall_relative_placement_int <= 2 - then 'Below/Far Below' - when overall_relative_placement_int = 3 - then 'Approaching' - when overall_relative_placement_int >= 4 - then 'At/Above' - end as iready_proficiency, - from {{ ref("int_iready__diagnostic_results") }} where rn_subj_round = 1 and test_round = 'EOY' ), @@ -202,6 +166,7 @@ with trim(split(specprog_name, '-')[offset(0)]) as bucket, trim(split(specprog_name, '-')[offset(1)]) as discipline, + from {{ ref("int_powerschool__spenrollments") }} where specprog_name like 'Bucket%' ), diff --git a/src/dbt/kipptaf/models/students/intermediate/properties/int_extracts__student_enrollments.yml b/src/dbt/kipptaf/models/students/intermediate/properties/int_extracts__student_enrollments.yml index fac927f55c..dbec87ce90 100644 --- a/src/dbt/kipptaf/models/students/intermediate/properties/int_extracts__student_enrollments.yml +++ b/src/dbt/kipptaf/models/students/intermediate/properties/int_extracts__student_enrollments.yml @@ -544,6 +544,8 @@ models: GPA band than their current unweighted GPA (cumulative_y1_gpa_unweighted). False if bands are equal, improving, or either GPA value is null. + - name: aligned_gender + data_type: string - name: enroll_status_string data_type: string - name: race_ethnicity