From 91005d437cdac72512961ec7fe9861bad3af8b26 Mon Sep 17 00:00:00 2001 From: ChungTingLao Date: Tue, 17 Jun 2025 14:19:42 +0100 Subject: [PATCH 1/5] dev-alt-XtoCityDB: added new example for OS building data. Instead of transforming building data into CityDB schema, this runs custom SQL scripts to extract thematic surfaces and make available through Ontop via custom mapping. It can also work with TOID-UPRN data. --- .../building-cambridge-vector-preprocess.sql | 145 ++++++++++++++++++ .../config/building-cambridge-vector.json | 42 +++++ ...ox-footprint-building-cambridge-vector.sql | 11 ++ .../building-cambridge-vector.obda | 101 ++++++++++++ .../building-cambridge/vector-toid-uprn.obda | 21 +++ 5 files changed, 320 insertions(+) create mode 100644 examples/datasets/inputs/config/building-cambridge-vector-preprocess.sql create mode 100644 examples/datasets/inputs/config/building-cambridge-vector.json create mode 100644 examples/datasets/inputs/config/mapbox-footprint-building-cambridge-vector.sql create mode 100644 examples/datasets/inputs/data/building-cambridge/building-cambridge-vector.obda create mode 100644 examples/datasets/inputs/data/building-cambridge/vector-toid-uprn.obda diff --git a/examples/datasets/inputs/config/building-cambridge-vector-preprocess.sql b/examples/datasets/inputs/config/building-cambridge-vector-preprocess.sql new file mode 100644 index 00000000..6bb05c37 --- /dev/null +++ b/examples/datasets/inputs/config/building-cambridge-vector-preprocess.sql @@ -0,0 +1,145 @@ +-- Extensions needed +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; +CREATE EXTENSION IF NOT EXISTS postgis_sfcgal; + +-- Create sequence for id, so no building and surface geometry will have the same id + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_sequences + WHERE schemaname = 'public' AND sequencename = 'shared_id_seq' + ) THEN + CREATE SEQUENCE shared_id_seq START 1; + END IF; +END $$; + +-- Create the "building" table (if not exists) +CREATE TABLE IF NOT EXISTS building ( + id INT PRIMARY KEY DEFAULT nextval('shared_id_seq'), + uuid UUID DEFAULT gen_random_uuid(), + os_topo_toid TEXT, + height DOUBLE PRECISION, + footprint_id INT +); + +-- Create the "surface_geometry" table (if not exists) +CREATE TABLE IF NOT EXISTS surface_geometry ( + id INT PRIMARY KEY DEFAULT nextval('shared_id_seq'), + uuid UUID DEFAULT gen_random_uuid(), + building_id INT NOT NULL, + surface_type TEXT, + geom GEOMETRY(MULTIPOLYGONZ) +); + +-- Add foreign key constraints (if not exists) +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.table_constraints + WHERE constraint_name = 'fk_building_footprint' + ) THEN + ALTER TABLE building + ADD CONSTRAINT fk_building_footprint FOREIGN KEY (footprint_id) + REFERENCES surface_geometry(id) ON DELETE SET NULL; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM information_schema.table_constraints + WHERE constraint_name = 'fk_surface_building' + ) THEN + ALTER TABLE surface_geometry + ADD CONSTRAINT fk_surface_building FOREIGN KEY (building_id) + REFERENCES building(id) ON DELETE CASCADE; + END IF; +END $$; + +-- Create spatial index for faster queries (if not exists) +CREATE INDEX IF NOT EXISTS idx_surface_geometry_geom ON surface_geometry USING GIST(geom); +CREATE INDEX IF NOT EXISTS idx_surface_geometry_building_id ON surface_geometry(building_id); + +-- Augment original data to have a footprint with elevation +ALTER TABLE "building-cambridge" ADD COLUMN "footprint" GEOMETRY(MULTIPOLYGONZ); + +UPDATE "building-cambridge" +SET "footprint" = ST_SetSRID( + ST_Translate(ST_Force3D("polygon"), 0, 0, "abshmin"), + ST_SRID("polygon") +); + +-- Create index for geometries +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE indexname = 'building-cambridge-footprint-index' + ) THEN + CREATE INDEX "building-cambridge-footprint-index" + ON "building-cambridge" USING GIST ("footprint"); + END IF; +END $$; + +-- Insert into building table +INSERT INTO building (os_topo_toid, height, footprint_id) +SELECT + "os_topo_toid", + COALESCE("absh2", "abshmax" - "abshmin", "relh2") AS height, + NULL AS footprint_id +FROM "building-cambridge" +WHERE "os_topo_toid" IS NOT NULL +AND NOT EXISTS ( + SELECT 1 FROM building b + WHERE b."os_topo_toid" = "building-cambridge"."os_topo_toid" +); + +-- Add footprint to surface geometry table +INSERT INTO "surface_geometry" (building_id, surface_type, geom) +SELECT + building.id AS building_id, + 'footprint' AS surface_type, + "building-cambridge"."footprint" AS geom +FROM building +JOIN "building-cambridge" ON building."os_topo_toid" = "building-cambridge"."os_topo_toid"; + +-- Add footprint ID back to building table +UPDATE building b +SET footprint_id = s.id +FROM surface_geometry s +WHERE + s.surface_type = 'footprint' + AND s.building_id = b.id; + +-- Extract building geometry using a CTE +WITH extruded AS ( + SELECT + building_id, + ST_Extrude(geom, 0.0, 0.0, height) AS geom, + ST_SRID(geom) AS srid + FROM "surface_geometry" JOIN "building" ON "surface_geometry"."building_id" = "building"."id" + WHERE surface_type = 'footprint' +) +-- Populate `"surface_geometry"` using precomputed geometries +INSERT INTO "surface_geometry" (building_id, surface_type, geom) +SELECT + building_id, + 'unknown', + ST_SetSRID((ST_Dump(geom)).geom, srid) +FROM extruded; + +-- Categorize surfaces based on height and elevation +UPDATE "surface_geometry" sg +SET surface_type = + CASE + WHEN ST_ZMin(sg.geom) = ST_ZMax(sg.geom) AND ST_ZMin(sg.geom) = ( + SELECT MIN(ST_ZMin(geom)) FROM "surface_geometry" WHERE building_id = sg.building_id + ) THEN 'ground' + WHEN ST_ZMin(sg.geom) = ST_ZMax(sg.geom) AND ST_ZMin(sg.geom) = ( + SELECT MAX(ST_ZMax(geom)) FROM "surface_geometry" WHERE building_id = sg.building_id + ) THEN 'roof' + ELSE 'wall' + END +WHERE sg.surface_type = 'unknown'; + +-- Drop table of original data + +DROP TABLE "building-cambridge" \ No newline at end of file diff --git a/examples/datasets/inputs/config/building-cambridge-vector.json b/examples/datasets/inputs/config/building-cambridge-vector.json new file mode 100644 index 00000000..293fd5a1 --- /dev/null +++ b/examples/datasets/inputs/config/building-cambridge-vector.json @@ -0,0 +1,42 @@ +{ + "database": "postgres", + "workspace": "the_world_avatar", + "datasetDirectory": "building-cambridge", + "dataSubsets": [ + { + "type": "Vector", + "skip": true, + "subdirectory": "vector", + "table": "building-cambridge", + "sql": "@/inputs/config/building-cambridge-vector-preprocess.sql", + "geoServerSettings": { + "virtualTable": { + "name": "mapbox_layer", + "sql": "@/inputs/config/mapbox-footprint-building-cambridge-vector.sql", + "escapeSql": false, + "geometry": { + "name": "geometry", + "type": "Polygon", + "srid": 27700 + } + }, + "defaultStyle": "polygon" + } + }, + { + "type": "tabular", + "skip": false, + "subdirectory": "tabular", + "table": "TOID_UPRN_MatchingTable", + "ogr2ogrOptions": { + "layerCreationOptions": { + "schema": "public" + } + } + } + ], + "mappings": [ + "building-cambridge-vector.obda", + "vector-toid-uprn.obda" + ] +} \ No newline at end of file diff --git a/examples/datasets/inputs/config/mapbox-footprint-building-cambridge-vector.sql b/examples/datasets/inputs/config/mapbox-footprint-building-cambridge-vector.sql new file mode 100644 index 00000000..d42ba670 --- /dev/null +++ b/examples/datasets/inputs/config/mapbox-footprint-building-cambridge-vector.sql @@ -0,0 +1,11 @@ +SELECT + "public"."building"."id" AS "building_id", + COALESCE("public"."building"."height", 100.0) AS "building_height", + "public"."surface_geometry"."geom" AS "geometry", + "public"."building"."os_topo_toid" AS "os_topo_toid", + 'https://theworldavatar.io/kg/Building/' || "public"."building"."uuid" AS "iri" +FROM + "public"."building" + JOIN "public"."surface_geometry" ON "public"."building"."id" = "public"."surface_geometry"."building_id" +WHERE + "public"."surface_geometry"."surface_type" = 'footprint' \ No newline at end of file diff --git a/examples/datasets/inputs/data/building-cambridge/building-cambridge-vector.obda b/examples/datasets/inputs/data/building-cambridge/building-cambridge-vector.obda new file mode 100644 index 00000000..9780db39 --- /dev/null +++ b/examples/datasets/inputs/data/building-cambridge/building-cambridge-vector.obda @@ -0,0 +1,101 @@ +[PrefixDeclaration] +owl: http://www.w3.org/2002/07/owl# +grp: http://www.opengis.net/citygml/cityobjectgroup/2.0/ +sf: http://www.opengis.net/ont/sf# +bldg: http://www.opengis.net/citygml/building/2.0/ +obda: https://w3id.org/obda/vocabulary# +xsd: http://www.w3.org/2001/XMLSchema# +rdfs: http://www.w3.org/2000/01/rdf-schema# +rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns# +www: http://www.w3.org/2001/ +geo: http://www.opengis.net/ont/geosparql# +city: http://cui.unige.ch/citygml/2.0/ +xml: http://www.w3.org/XML/1998/namespace +twa: https://theworldavatar.io/kg/ + +[MappingDeclaration] @collection [[ + +mappingId Non-Solid Geometry +target city:geometry/{"id"} a sf:Polygon ; geo:asWKT " {geometry_wkt}"^^geo:wktLiteral ; geo:hasMetricArea {"area_sqm"}^^www:XMLSchema#double . +source SELECT "id", ST_ASTEXT(ST_TRANSFORM("geom", 4326)) AS "geometry_wkt", ST_AREA("geom") AS "area_sqm" + FROM "public"."surface_geometry" + +mappingId Surface Geometry - Parent +target city:geometry/{"id"} grp:parent city:geometry/{"parent_id"} . +source SELECT "id", "id" AS "parent_id" + FROM "public"."surface_geometry" + +mappingId Building +target twa:Building/{uuid} obda:isCanonicalIRIOf city:Building/{"id"} . + city:Building/{"id"} a bldg:Building ; bldg:measuredHeight {"measured_height"}^^www:XMLSchema#double ; bldg:class {"class"}^^www:XMLSchema#string . +source SELECT "id", "uuid", "height" AS "measured_height", 26 AS "class" + FROM "public"."building" + +mappingId Building - LoDFootprint +target city:Building/{"id"} bldg:lod0FootPrint city:geometry/{"lod0_footprint_id"} . +source SELECT "id", "footprint_id" AS "lod0_footprint_id" + FROM "public"."building" + +mappingId Surface Geometry to Building +target city:Building/{"cityobject_id"} geo:hasGeometry city:geometry/{"footprint_id"} . +source SELECT "id" AS "cityobject_id", "footprint_id" + FROM "public"."building" + +mappingId BuildingRoofSurface +target twa:RoofSurface/{uuid} obda:isCanonicalIRIOf city:RoofSurface/{"id"} . + city:RoofSurface/{"id"} a bldg:RoofSurface . +source SELECT id, uuid + FROM "public"."surface_geometry" + WHERE surface_type='roof' + +mappingId BuildingGroundSurface +target twa:GroundSurface/{uuid} obda:isCanonicalIRIOf city:GroundSurface/{"id"} . + city:GroundSurface/{"id"} a bldg:GroundSurface . +source SELECT id, uuid + FROM "public"."surface_geometry" + WHERE surface_type='ground' + +mappingId BuildingWallSurface +target twa:WallSurface/{uuid} obda:isCanonicalIRIOf city:WallSurface/{"id"} . + city:WallSurface/{"id"} a bldg:WallSurface . +source SELECT id, uuid + FROM "public"."surface_geometry" + WHERE surface_type='wall' + +mappingId Surface Geometry to City Object - roof surface +target city:RoofSurface/{"cityobject_id"} geo:hasGeometry city:geometry/{"id"} . +source SELECT "id" AS "cityobject_id", "id" + FROM "public"."surface_geometry" + WHERE "surface_type" = 'roof' + +mappingId Surface Geometry to City Object - ground surface +target city:GroundSurface/{"cityobject_id"} geo:hasGeometry city:geometry/{"id"} . +source SELECT "id" AS "cityobject_id", "id" + FROM "public"."surface_geometry" + WHERE "surface_type" = 'ground' + +mappingId Surface Geometry to City Object - wall surface +target city:WallSurface/{"cityobject_id"} geo:hasGeometry city:geometry/{"id"} . +source SELECT "id" AS "cityobject_id", "id" + FROM "public"."surface_geometry" + WHERE "surface_type" = 'wall' + +mappingId Building bounded by Roof surface +target city:Building/{"building_id"} bldg:boundedBy city:RoofSurface/{"thematic_surface_id"} . +source SELECT building_id, id AS thematic_surface_id + FROM "public"."surface_geometry" + WHERE "surface_type" = 'roof' + +mappingId Building bounded by Ground surface +target city:Building/{"building_id"} bldg:boundedBy city:GroundSurface/{"thematic_surface_id"} . +source SELECT building_id, id AS thematic_surface_id + FROM "public"."surface_geometry" + WHERE "surface_type" = 'ground' + +mappingId Building bounded by Wall surface +target city:Building/{"building_id"} bldg:boundedBy city:WallSurface/{"thematic_surface_id"} . +source SELECT building_id, id AS thematic_surface_id + FROM "public"."surface_geometry" + WHERE "surface_type" = 'wall' + +]] \ No newline at end of file diff --git a/examples/datasets/inputs/data/building-cambridge/vector-toid-uprn.obda b/examples/datasets/inputs/data/building-cambridge/vector-toid-uprn.obda new file mode 100644 index 00000000..5f53d237 --- /dev/null +++ b/examples/datasets/inputs/data/building-cambridge/vector-toid-uprn.obda @@ -0,0 +1,21 @@ +[PrefixDeclaration] +building: http://theworldavatar.io/BuildingWallSurface/ +os: http://data.ordnancesurvey.co.uk/ontology/spatialrelations/ +owl: http://www.w3.org/2002/07/owl# +rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns# +xml: http://www.w3.org/XML/1998/namespace +xsd: http://www.w3.org/2001/XMLSchema# +obda: https://w3id.org/obda/vocabulary# +rdfs: http://www.w3.org/2000/01/rdf-schema# +city: http://cui.unige.ch/citygml/2.0/ + +[MappingDeclaration] @collection [[ +mappingId Asset-mapping-TOID-UPRN +target city:Building/{"id"} os:hasTOID {os_topo_toid}^^xsd:string ; + os:hasUPRN {uprn}^^xsd:integer . +source SELECT "public"."building"."id", "public"."building"."os_topo_toid", "public"."TOID_UPRN_MatchingTable"."IDENTIFIER_1" as uprn + FROM "public"."building" JOIN "public"."TOID_UPRN_MatchingTable" ON + "public"."building"."os_topo_toid" = "public"."TOID_UPRN_MatchingTable"."IDENTIFIER_2" + + +]] \ No newline at end of file From 2e15a9aa052e8a67543a14b304f22e8231eb02ad Mon Sep 17 00:00:00 2001 From: ChungTingLao Date: Wed, 25 Jun 2025 10:30:54 +0100 Subject: [PATCH 2/5] dev-alt-XtoCityDB: changed version numbers of stack. --- stack-clients/docker-compose.yml | 2 +- stack-clients/pom.xml | 2 +- stack-data-uploader/docker-compose.yml | 2 +- stack-data-uploader/pom.xml | 4 ++-- stack-manager/docker-compose.yml | 2 +- stack-manager/pom.xml | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/stack-clients/docker-compose.yml b/stack-clients/docker-compose.yml index af3df652..48623cc9 100644 --- a/stack-clients/docker-compose.yml +++ b/stack-clients/docker-compose.yml @@ -1,6 +1,6 @@ services: stack-client: - image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.48.0 + image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.48.1-XtoCityDB-SNAPSHOT secrets: - blazegraph_password - postgis_password diff --git a/stack-clients/pom.xml b/stack-clients/pom.xml index 968c1963..e9ba8244 100644 --- a/stack-clients/pom.xml +++ b/stack-clients/pom.xml @@ -7,7 +7,7 @@ com.cmclinnovations stack-clients - 1.48.0 + 1.48.0-XtoCityDB-SNAPSHOT Stack Clients https://theworldavatar.io diff --git a/stack-data-uploader/docker-compose.yml b/stack-data-uploader/docker-compose.yml index 351cb8ca..51d05ade 100644 --- a/stack-data-uploader/docker-compose.yml +++ b/stack-data-uploader/docker-compose.yml @@ -1,6 +1,6 @@ services: stack-data-uploader: - image: ghcr.io/theworldavatar/stack-data-uploader${IMAGE_SUFFIX}:1.48.0 + image: ghcr.io/theworldavatar/stack-data-uploader${IMAGE_SUFFIX}:1.48.1-XtoCityDB-SNAPSHOT secrets: - blazegraph_password - postgis_password diff --git a/stack-data-uploader/pom.xml b/stack-data-uploader/pom.xml index 5ba140f3..106ca098 100644 --- a/stack-data-uploader/pom.xml +++ b/stack-data-uploader/pom.xml @@ -7,7 +7,7 @@ com.cmclinnovations stack-data-uploader - 1.48.0 + 1.48.0-XtoCityDB-SNAPSHOT Stack Data Uploader https://theworldavatar.io @@ -38,7 +38,7 @@ com.cmclinnovations stack-clients - 1.48.0 + 1.48.0-XtoCityDB-SNAPSHOT diff --git a/stack-manager/docker-compose.yml b/stack-manager/docker-compose.yml index d346614c..f2fda2c1 100644 --- a/stack-manager/docker-compose.yml +++ b/stack-manager/docker-compose.yml @@ -1,6 +1,6 @@ services: stack-manager: - image: ghcr.io/theworldavatar/stack-manager${IMAGE_SUFFIX}:1.48.0 + image: ghcr.io/theworldavatar/stack-manager${IMAGE_SUFFIX}:1.48.1-XtoCityDB-SNAPSHOT environment: EXTERNAL_PORT: "${EXTERNAL_PORT-3838}" STACK_BASE_DIR: "${STACK_BASE_DIR}" diff --git a/stack-manager/pom.xml b/stack-manager/pom.xml index 9453dd27..947e6597 100644 --- a/stack-manager/pom.xml +++ b/stack-manager/pom.xml @@ -7,7 +7,7 @@ com.cmclinnovations stack-manager - 1.48.0 + 1.48.0-XtoCityDB-SNAPSHOT Stack Manager https://theworldavatar.io @@ -38,7 +38,7 @@ com.cmclinnovations stack-clients - 1.48.0 + 1.48.0-XtoCityDB-SNAPSHOT From 77f06e30b189ea13f3aab1b56daec58ce2172d8a Mon Sep 17 00:00:00 2001 From: ChungTingLao Date: Wed, 25 Jun 2025 10:50:01 +0100 Subject: [PATCH 3/5] dev-alt-XtoCityDB: directly insert footprint in XtoCityDB instead of relying on CityDB post-processing. --- .../clients/citydb/citydb_populate_citydb.sql | 133 +++++++++++++++++- .../citydb/citydb_preprocess_building.sql | 23 ++- 2 files changed, 150 insertions(+), 6 deletions(-) diff --git a/stack-clients/src/main/resources/com/cmclinnovations/stack/clients/citydb/citydb_populate_citydb.sql b/stack-clients/src/main/resources/com/cmclinnovations/stack/clients/citydb/citydb_populate_citydb.sql index 372c1675..aa6af87a 100644 --- a/stack-clients/src/main/resources/com/cmclinnovations/stack/clients/citydb/citydb_populate_citydb.sql +++ b/stack-clients/src/main/resources/com/cmclinnovations/stack/clients/citydb/citydb_populate_citydb.sql @@ -2,6 +2,11 @@ SET search_path TO public, citydb; +DROP TABLE IF EXISTS "public"."building_without_footprint_CityDB", +"public"."precise_footprint_CityDB", +"public"."footprint_soup_CityDB", +"public"."fp_parent"; + INSERT INTO "citydb"."cityobject" ( "objectclass_id", @@ -198,6 +203,130 @@ SET WHERE "root_id" IS NULL; -DROP TABLE IF EXISTS "public"."raw_building_XtoCityDB"; +-- create footprint +CREATE TABLE public."building_without_footprint_CityDB" AS ( + SELECT + c.id, + c.gmlid, + ST_Zmin(c.envelope) AS zmin + FROM + cityobject AS c + JOIN "public"."raw_building_XtoCityDB" AS r ON c.gmlid = r.gmlid +); + +CREATE TABLE public."precise_footprint_CityDB" AS ( + SELECT + b.id AS building_id, + ST_Union( + ST_MakeValid(ST_Force2D(s.geom)), + 0.00000001 + ) AS U + FROM + "public"."raw_surface_XtoCityDB" AS s + JOIN "public"."building_without_footprint_CityDB" AS b ON b.gmlid = s.building_gmlid + WHERE + s.class = 35 + GROUP BY + building_id +); + +CREATE TABLE public."footprint_soup_CityDB" AS ( + SELECT + b.id, + b.gmlid, + ST_Force3D( + ( + ST_Dump(p.U) + ).geom, + b.zmin + ) AS geometry + FROM + "public"."building_without_footprint_CityDB" AS b + LEFT JOIN public."precise_footprint_CityDB" AS p ON b.id = p.building_id +); + +WITH temp AS ( + INSERT INTO + citydb.surface_geometry ( + gmlid, + is_solid, + is_composite, + is_triangulated, + is_xlink, + is_reverse, + cityobject_id + ) ( + SELECT + b.gmlid || '_footprint', + 0, + 0, + 0, + 0, + 0, + b.id + FROM + "public"."building_without_footprint_CityDB" AS b + ) RETURNING id AS footprint_id, + cityobject_id AS bid, + gmlid +) +SELECT + footprint_id, + bid, + gmlid INTO TABLE public.fp_parent +FROM + temp; + +UPDATE + building AS b +SET + lod0_footprint_id = f.footprint_id +FROM + fp_parent AS f +WHERE + b.id = f.bid; + +INSERT INTO + citydb.surface_geometry ( + gmlid, + parent_id, + root_id, + is_solid, + is_composite, + is_triangulated, + is_xlink, + is_reverse, + geometry, + cityobject_id + ) ( + SELECT + s.gmlid || '_footprint_child', + f.footprint_id, + f.footprint_id, + 0, + 0, + 0, + 0, + 0, + s.geometry, + s.id + FROM + public."footprint_soup_CityDB" AS s + JOIN public.fp_parent AS f ON s.id = f.bid + WHERE + GeometryType(geometry) = 'POLYGON' + ); + +UPDATE + surface_geometry +SET + root_id = id +WHERE + root_id IS NULL; -DROP TABLE IF EXISTS "public"."raw_surface_XtoCityDB"; \ No newline at end of file +DROP TABLE IF EXISTS "public"."raw_building_XtoCityDB", +"public"."raw_surface_XtoCityDB", +"public"."building_without_footprint_CityDB", +"public"."precise_footprint_CityDB", +"public"."footprint_soup_CityDB", +"public"."fp_parent"; \ No newline at end of file diff --git a/stack-clients/src/main/resources/com/cmclinnovations/stack/clients/citydb/citydb_preprocess_building.sql b/stack-clients/src/main/resources/com/cmclinnovations/stack/clients/citydb/citydb_preprocess_building.sql index c7bb9f90..166ded8d 100644 --- a/stack-clients/src/main/resources/com/cmclinnovations/stack/clients/citydb/citydb_preprocess_building.sql +++ b/stack-clients/src/main/resources/com/cmclinnovations/stack/clients/citydb/citydb_preprocess_building.sql @@ -2,9 +2,8 @@ SET search_path TO public, citydb; -DROP TABLE IF EXISTS "public"."raw_building_XtoCityDB"; - -DROP TABLE IF EXISTS "public"."raw_surface_XtoCityDB"; +DROP TABLE IF EXISTS "public"."raw_building_XtoCityDB", +"public"."raw_surface_XtoCityDB"; CREATE TABLE "public"."raw_building_XtoCityDB" AS ( SELECT @@ -24,6 +23,14 @@ CREATE TABLE "public"."raw_building_XtoCityDB" AS ( AND "{height}" IS NOT NULL ); +CREATE INDEX IF NOT EXISTS idx_raw_building_gmlid ON "public"."raw_building_XtoCityDB" ("gmlid"); + +CREATE INDEX IF NOT EXISTS idx_raw_building_oldid ON "public"."raw_building_XtoCityDB" ("{IDval}"); + +CREATE INDEX IF NOT EXISTS idx_raw_building_geom ON "public"."raw_building_XtoCityDB" USING GIST ("geom"); + +ANALYZE "public"."raw_building_XtoCityDB"; + CREATE TABLE "public"."raw_surface_XtoCityDB" AS ( SELECT "building_gmlid", @@ -52,4 +59,12 @@ CREATE TABLE "public"."raw_surface_XtoCityDB" AS ( "public"."raw_building_XtoCityDB" ) AS "table1" ) AS "table2" -); \ No newline at end of file +); + +CREATE INDEX IF NOT EXISTS idx_raw_surface_gmlid ON "public"."raw_surface_XtoCityDB" ("gmlid"); + +CREATE INDEX IF NOT EXISTS idx_raw_surface_building_gmlid ON "public"."raw_surface_XtoCityDB" ("building_gmlid"); + +CREATE INDEX IF NOT EXISTS idx_raw_surface_geom ON "public"."raw_surface_XtoCityDB" USING GIST ("geom"); + +ANALYZE "public"."raw_surface_XtoCityDB"; From 8461bcd19c46c25827c7a537ee9302514d14ea7d Mon Sep 17 00:00:00 2001 From: ChungTingLao Date: Wed, 25 Jun 2025 12:05:14 +0100 Subject: [PATCH 4/5] dev-alt-XtoCityDB: allow user to control whether postgres tables are unlogged for CityDB post-processing. --- .../stack/clients/core/datasets/CityDB.java | 14 ++++++++++---- .../stack/clients/core/datasets/XtoCityDB.java | 9 +++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/CityDB.java b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/CityDB.java index d802ea76..4a3aa64a 100644 --- a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/CityDB.java +++ b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/CityDB.java @@ -37,6 +37,8 @@ public class CityDB extends GeoServerDataSubset { @JsonProperty private boolean augmentData = true; @JsonProperty + private boolean unlogWhenAugment = false; + @JsonProperty private boolean discoverThematicSurface = false; @JsonProperty private double critAreaRatio = 0.1; @@ -109,8 +111,10 @@ protected void loadDataInternal(Path dataSubsetDir, String database, String base protected void augmentData(String database) { - logger.info("Setting tables to unlogged for better write performance..."); - CityDBClient.getInstance().unlogTable(database); + if (unlogWhenAugment) { + logger.info("Setting tables to unlogged for better write performance..."); + CityDBClient.getInstance().unlogTable(database); + } if (discoverThematicSurface) { logger.info("Discovering thematic surface..."); @@ -121,8 +125,10 @@ protected void augmentData(String database) { logger.info("Adding building footprint..."); CityDBClient.getInstance().addFootprint(database); - logger.info("Setting tables to logged..."); - CityDBClient.getInstance().relogTable(database); + if (unlogWhenAugment) { + logger.info("Setting tables to logged..."); + CityDBClient.getInstance().relogTable(database); + } } @Override diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/XtoCityDB.java b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/XtoCityDB.java index f5045369..6d688b40 100644 --- a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/XtoCityDB.java +++ b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/XtoCityDB.java @@ -4,6 +4,9 @@ import java.util.HashMap; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.cmclinnovations.stack.clients.citydb.CityDBClient; import com.cmclinnovations.stack.clients.gdal.GDALClient; import com.cmclinnovations.stack.clients.gdal.Ogr2OgrOptions; @@ -12,6 +15,8 @@ public class XtoCityDB extends CityDB { + private static final Logger logger = LoggerFactory.getLogger(XtoCityDB.class); + @JsonProperty private Ogr2OgrOptions ogr2ogrOptions = new Ogr2OgrOptions(); @@ -31,13 +36,17 @@ public class XtoCityDB extends CityDB { protected void loadDataInternal(Path dataSubsetDir, String database, String baseIRI, String lineage) { setPreviousFile(dataSubsetDir.resolveSibling(dataSubsetDir.getFileName() + "_previous") .resolve("previous.gz")); + logger.info("Uploading original data to PostGIS..."); GDALClient.getInstance() .uploadVectorFilesToPostGIS(database, getSchema(), getTable(), dataSubsetDir.toString(), ogr2ogrOptions, false); + logger.info("Initialising CityDB schema..."); CityDBClient.getInstance() .updateDatabase(database, getSridIn()); + logger.info("Preparing original data..."); CityDBClient.getInstance().preparePGforCityDB(database, getTable(), JsonHelper.handleFileValues(preprocessSql), minArea, columnMap); + logger.info("Inserting processed data into CityDB schema..."); CityDBClient.getInstance().populateCityDBbySQL(database, lineage, columnMap); CityDBClient.getInstance().addIRIs(database, baseIRI); } From d6d63fbb655f4580d4ddc9ce5d9395348ebe7234 Mon Sep 17 00:00:00 2001 From: ChungTingLao Date: Wed, 25 Jun 2025 14:08:28 +0100 Subject: [PATCH 5/5] dev-alt-X-toCityDB: changed default option to preserve original behaviour for CityDB datasets. --- .../com/cmclinnovations/stack/clients/core/datasets/CityDB.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/CityDB.java b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/CityDB.java index 4a3aa64a..f05be87c 100644 --- a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/CityDB.java +++ b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/CityDB.java @@ -37,7 +37,7 @@ public class CityDB extends GeoServerDataSubset { @JsonProperty private boolean augmentData = true; @JsonProperty - private boolean unlogWhenAugment = false; + private boolean unlogWhenAugment = true; @JsonProperty private boolean discoverThematicSurface = false; @JsonProperty