add geometry repair to pluto

.github/workflows/ingest_single.yml

@@ -7,7 +7,7 @@ on:
       dataset:
         description: "Name of the dataset (required)"
         required: true
-        default: dcp_mappluto
+        default: dcp_mappluto_wi
       latest:
         type: boolean
         description: "Tag this version as latest (optional)"

.github/workflows/pluto_build.yml

@@ -83,18 +83,9 @@ jobs:
       run: python -m dcpy.lifecycle.builds.load load --recipe-path ${{ inputs.recipe_file
         }}.lock.yml
 
-    - name: Load Local Data
-      run: ./01_load_local_csvs.sh
-
-    - name: building ...
+    - name: Build
       run: ./02_build.sh
 
-    - name: apply corrections
-      run: ./03_corrections.sh
-
-    - name: Archive
-      run: ./04_archive.sh
-
     - name: QAQC
       run: ./05_qaqc.sh
 

products/pluto/models/_sources.yml

@@ -57,6 +57,7 @@ sources:
       records per a condominium.
 
   - name: previous_pluto
+  - name: pluto_geom
   - name: export_pluto
     description: final PLUTO table
   - name: dcp_zoningdistricts

products/pluto/pluto_build/02_build.sh

@@ -3,6 +3,7 @@ source ./bash/config.sh
 set_error_traps
 
 echo "Starting to build PLUTO ..."
+run_sql_file sql/_create.sql
 run_sql_file sql/preprocessing.sql
 run_sql_file sql/create_pts.sql
 run_sql_file sql/create_rpad_geo.sql
@@ -108,4 +109,44 @@ run_sql_file sql/plutomapid_1.sql
 run_sql_file sql/plutomapid_2.sql
 run_sql_file sql/shorelineclip.sql
 
+run_sql_file sql/corr_create.sql
+
+echo "Applying corrections to PLUTO"
+run_sql_file sql/corr_lotarea.sql
+run_sql_file sql/corr_template.sql -v FIELD=yearbuilt
+run_sql_file sql/corr_template.sql -v FIELD=ownername
+run_sql_file sql/corr_ownername_punctuation.sql
+run_sql_file sql/corr_template.sql -v FIELD=cd
+run_sql_file sql/corr_template.sql -v FIELD=numfloors
+run_sql_file sql/corr_template.sql -v FIELD=numbldgs
+run_sql_file sql/corr_template.sql -v FIELD=unitsres
+run_sql_file sql/corr_template.sql -v FIELD=unitstotal
+run_sql_file sql/corr_inwoodrezoning.sql
+run_sql_file sql/corr_template.sql -v FIELD=bct2020
+run_sql_file sql/corr_template.sql -v FIELD=address
+run_sql_file sql/remove_unitlots.sql
+
+echo "Creating export tables"
+run_sql_file sql/export.sql
+
+run_sql_file \
+  sql/export_mappluto_shp.sql\
+  -v TABLE='mappluto'\
+  -v GEOM='clipped_2263'
+
+run_sql_file \
+  sql/export_mappluto_shp.sql\
+  -v TABLE='mappluto_unclipped'\
+  -v GEOM='geom_2263'
+
+run_sql_file \
+  sql/export_mappluto_gdb.sql\
+  -v TABLE='mappluto_gdb'\
+  -v GEOM='clipped_2263'
+
+run_sql_file \
+  sql/export_mappluto_gdb.sql\
+  -v TABLE='mappluto_unclipped_gdb'\
+  -v GEOM='geom_2263'
+
 echo 'Done'

products/pluto/tests/assert_all_geometry_is_valid.sql

@@ -0,0 +1,56 @@
+{{ 
+    config(
+        tags = ['de_check', 'minor', 'major'], 
+        meta = {
+            'description': '''
+				This test checks for any invalid lot geometries
+			''',
+            'next_steps': 'Contact DE and GIS to investigate'
+        }
+    ) 
+}}
+
+WITH pluto_geom AS (
+    SELECT
+        bbl,
+        geom_2263,
+        geom_4326,
+        clipped_2263,
+        clipped_4326
+    FROM {{ source('build_sources', 'pluto_geom') }}
+),
+
+-- ST_IsValidDetail returns a row containing:
+--      a boolean (valid) stating if a geometry is valid
+--      a varchar (reason) stating why it is invalid
+--      a geometry (location) pointing out where it is invalid
+-- https://postgis.net/docs/ST_IsValidDetail.html
+validity_details AS (
+    SELECT
+        bbl,
+        ST_ISVALIDDETAIL(geom_2263) AS validity_details_geom_2263,
+        ST_ISVALIDDETAIL(geom_4326) AS validity_details_geom_4326,
+        ST_ISVALIDDETAIL(clipped_2263) AS validity_details_clipped_2263,
+        ST_ISVALIDDETAIL(clipped_4326) AS validity_details_clipped_4326
+    FROM pluto_geom
+),
+
+check_all_geoms AS (
+    SELECT
+        *,
+        (
+            false
+            = ANY(
+                ARRAY[
+                    (validity_details_geom_2263).valid,
+                    (validity_details_geom_4326).valid,
+                    (validity_details_clipped_2263).valid,
+                    (validity_details_clipped_4326).valid
+                ]
+            )
+        ) AS any_geoms_invalid
+    FROM validity_details
+)
+
+SELECT * FROM check_all_geoms
+WHERE any_geoms_invalid