XtoCityDB improvement and alternative

examples/datasets/inputs/config/building-cambridge-vector-preprocess.sql

@@ -0,0 +1,145 @@
+-- Extensions needed
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+CREATE EXTENSION IF NOT EXISTS postgis_sfcgal;
+
+-- Create sequence for id, so no building and surface geometry will have the same id
+
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM pg_sequences
+        WHERE schemaname = 'public' AND sequencename = 'shared_id_seq'
+    ) THEN
+        CREATE SEQUENCE shared_id_seq START 1;
+    END IF;
+END $$;
+
+-- Create the "building" table (if not exists)
+CREATE TABLE IF NOT EXISTS building (
+    id INT PRIMARY KEY DEFAULT nextval('shared_id_seq'),
+    uuid UUID DEFAULT gen_random_uuid(),
+    os_topo_toid TEXT,
+    height DOUBLE PRECISION,
+    footprint_id INT
+);
+
+-- Create the "surface_geometry" table (if not exists)
+CREATE TABLE IF NOT EXISTS surface_geometry (
+    id INT PRIMARY KEY DEFAULT nextval('shared_id_seq'),
+    uuid UUID DEFAULT gen_random_uuid(),
+    building_id INT NOT NULL,
+    surface_type TEXT,
+    geom GEOMETRY(MULTIPOLYGONZ)
+);
+
+-- Add foreign key constraints (if not exists)
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM information_schema.table_constraints
+        WHERE constraint_name = 'fk_building_footprint'
+    ) THEN
+        ALTER TABLE building
+        ADD CONSTRAINT fk_building_footprint FOREIGN KEY (footprint_id)
+        REFERENCES surface_geometry(id) ON DELETE SET NULL;
+    END IF;
+
+    IF NOT EXISTS (
+        SELECT 1 FROM information_schema.table_constraints
+        WHERE constraint_name = 'fk_surface_building'
+    ) THEN
+        ALTER TABLE surface_geometry
+        ADD CONSTRAINT fk_surface_building FOREIGN KEY (building_id)
+        REFERENCES building(id) ON DELETE CASCADE;
+    END IF;
+END $$;
+
+-- Create spatial index for faster queries (if not exists)
+CREATE INDEX IF NOT EXISTS idx_surface_geometry_geom ON surface_geometry USING GIST(geom);
+CREATE INDEX IF NOT EXISTS idx_surface_geometry_building_id ON surface_geometry(building_id);
+
+-- Augment original data to have a footprint with elevation
+ALTER TABLE "building-cambridge" ADD COLUMN "footprint" GEOMETRY(MULTIPOLYGONZ);
+
+UPDATE "building-cambridge"
+SET "footprint" = ST_SetSRID(
+    ST_Translate(ST_Force3D("polygon"), 0, 0, "abshmin"),
+    ST_SRID("polygon")
+);
+
+-- Create index for geometries
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM pg_indexes
+        WHERE indexname = 'building-cambridge-footprint-index'
+    ) THEN
+        CREATE INDEX "building-cambridge-footprint-index"
+        ON "building-cambridge" USING GIST ("footprint");
+    END IF;
+END $$;
+
+-- Insert into building table
+INSERT INTO building (os_topo_toid, height, footprint_id)
+SELECT
+    "os_topo_toid",
+    COALESCE("absh2", "abshmax" - "abshmin", "relh2") AS height,
+    NULL AS footprint_id
+FROM "building-cambridge"
+WHERE "os_topo_toid" IS NOT NULL
+AND NOT EXISTS (
+    SELECT 1 FROM building b
+    WHERE b."os_topo_toid" = "building-cambridge"."os_topo_toid"
+);
+
+-- Add footprint to surface geometry table
+INSERT INTO "surface_geometry" (building_id, surface_type, geom)
+SELECT
+    building.id AS building_id,
+    'footprint' AS surface_type,
+    "building-cambridge"."footprint" AS geom
+FROM building
+JOIN "building-cambridge" ON building."os_topo_toid" = "building-cambridge"."os_topo_toid";
+
+-- Add footprint ID back to building table
+UPDATE building b
+SET footprint_id = s.id
+FROM surface_geometry s
+WHERE
+    s.surface_type = 'footprint'
+    AND s.building_id = b.id;
+
+-- Extract building geometry using a CTE
+WITH extruded AS (
+    SELECT
+        building_id,
+        ST_Extrude(geom, 0.0, 0.0, height) AS geom,
+        ST_SRID(geom) AS srid
+    FROM "surface_geometry" JOIN "building" ON "surface_geometry"."building_id" = "building"."id"
+    WHERE surface_type = 'footprint'
+)
+-- Populate `"surface_geometry"` using precomputed geometries
+INSERT INTO "surface_geometry" (building_id, surface_type, geom)
+SELECT
+    building_id,
+    'unknown',
+    ST_SetSRID((ST_Dump(geom)).geom, srid)
+FROM extruded;
+
+-- Categorize surfaces based on height and elevation
+UPDATE "surface_geometry" sg
+SET surface_type = 
+    CASE 
+        WHEN ST_ZMin(sg.geom) = ST_ZMax(sg.geom) AND ST_ZMin(sg.geom) = (
+            SELECT MIN(ST_ZMin(geom)) FROM "surface_geometry" WHERE building_id = sg.building_id
+        ) THEN 'ground'
+        WHEN ST_ZMin(sg.geom) = ST_ZMax(sg.geom) AND ST_ZMin(sg.geom) = (
+            SELECT MAX(ST_ZMax(geom)) FROM "surface_geometry" WHERE building_id = sg.building_id
+        ) THEN 'roof'
+        ELSE 'wall'
+    END
+WHERE sg.surface_type = 'unknown';
+
+-- Drop table of original data
+
+DROP TABLE "building-cambridge"

examples/datasets/inputs/config/building-cambridge-vector.json

@@ -0,0 +1,42 @@
+{
+    "database": "postgres",
+    "workspace": "the_world_avatar",
+    "datasetDirectory": "building-cambridge",
+    "dataSubsets": [
+        {
+            "type": "Vector",
+            "skip": true,
+            "subdirectory": "vector",
+            "table": "building-cambridge",
+            "sql": "@/inputs/config/building-cambridge-vector-preprocess.sql",
+            "geoServerSettings": {
+                "virtualTable": {
+                    "name": "mapbox_layer",
+                    "sql": "@/inputs/config/mapbox-footprint-building-cambridge-vector.sql",
+                    "escapeSql": false,
+                    "geometry": {
+                        "name": "geometry",
+                        "type": "Polygon",
+                        "srid": 27700
+                    }
+                },
+                "defaultStyle": "polygon"
+            }
+        },
+        {
+            "type": "tabular",
+            "skip": false,
+            "subdirectory": "tabular",
+            "table": "TOID_UPRN_MatchingTable",
+            "ogr2ogrOptions": {
+                "layerCreationOptions": {
+                    "schema": "public"
+                }
+            }
+        }
+    ],
+    "mappings": [
+        "building-cambridge-vector.obda",
+        "vector-toid-uprn.obda"
+    ]
+}

examples/datasets/inputs/config/mapbox-footprint-building-cambridge-vector.sql

@@ -0,0 +1,11 @@
+SELECT
+    "public"."building"."id" AS "building_id",
+    COALESCE("public"."building"."height", 100.0) AS "building_height",
+    "public"."surface_geometry"."geom" AS "geometry",
+    "public"."building"."os_topo_toid" AS "os_topo_toid",
+    'https://theworldavatar.io/kg/Building/' || "public"."building"."uuid" AS "iri"
+FROM
+    "public"."building"
+    JOIN "public"."surface_geometry" ON "public"."building"."id" = "public"."surface_geometry"."building_id"
+WHERE
+    "public"."surface_geometry"."surface_type" = 'footprint'

examples/datasets/inputs/data/building-cambridge/building-cambridge-vector.obda

@@ -0,0 +1,101 @@
+[PrefixDeclaration]
+owl:		http://www.w3.org/2002/07/owl#
+grp:		http://www.opengis.net/citygml/cityobjectgroup/2.0/
+sf:		http://www.opengis.net/ont/sf#
+bldg:		http://www.opengis.net/citygml/building/2.0/
+obda:		https://w3id.org/obda/vocabulary#
+xsd:		http://www.w3.org/2001/XMLSchema#
+rdfs:		http://www.w3.org/2000/01/rdf-schema#
+rdf:		http://www.w3.org/1999/02/22-rdf-syntax-ns#
+www:		http://www.w3.org/2001/
+geo:		http://www.opengis.net/ont/geosparql#
+city:		http://cui.unige.ch/citygml/2.0/
+xml:		http://www.w3.org/XML/1998/namespace
+twa:        https://theworldavatar.io/kg/
+
+[MappingDeclaration] @collection [[
+
+mappingId	Non-Solid Geometry
+target		city:geometry/{"id"} a sf:Polygon ; geo:asWKT "<http://www.opengis.net/def/crs/OGC/1.3/CRS84> {geometry_wkt}"^^geo:wktLiteral ; geo:hasMetricArea {"area_sqm"}^^www:XMLSchema#double . 
+source		SELECT "id", ST_ASTEXT(ST_TRANSFORM("geom", 4326)) AS "geometry_wkt", ST_AREA("geom") AS "area_sqm"
+			FROM "public"."surface_geometry"
+
+mappingId	Surface Geometry - Parent
+target		city:geometry/{"id"} grp:parent city:geometry/{"parent_id"} . 
+source		SELECT "id", "id" AS "parent_id"
+			FROM "public"."surface_geometry"
+
+mappingId	Building
+target		twa:Building/{uuid} obda:isCanonicalIRIOf city:Building/{"id"} .
+            city:Building/{"id"} a bldg:Building ; bldg:measuredHeight {"measured_height"}^^www:XMLSchema#double ; bldg:class {"class"}^^www:XMLSchema#string . 
+source		SELECT "id", "uuid", "height" AS "measured_height", 26 AS "class"
+            FROM "public"."building"
+
+mappingId	Building - LoDFootprint
+target		city:Building/{"id"} bldg:lod0FootPrint city:geometry/{"lod0_footprint_id"} . 
+source		SELECT "id", "footprint_id" AS "lod0_footprint_id"
+			FROM "public"."building"
+
+mappingId	Surface Geometry to Building
+target		city:Building/{"cityobject_id"} geo:hasGeometry city:geometry/{"footprint_id"} . 
+source		SELECT "id" AS "cityobject_id", "footprint_id"
+			FROM "public"."building"
+
+mappingId	BuildingRoofSurface
+target		twa:RoofSurface/{uuid} obda:isCanonicalIRIOf city:RoofSurface/{"id"} .
+            city:RoofSurface/{"id"} a bldg:RoofSurface . 
+source		SELECT id, uuid
+			FROM "public"."surface_geometry"
+			WHERE surface_type='roof'
+
+mappingId	BuildingGroundSurface
+target		twa:GroundSurface/{uuid} obda:isCanonicalIRIOf city:GroundSurface/{"id"} .
+            city:GroundSurface/{"id"} a bldg:GroundSurface . 
+source		SELECT id, uuid
+			FROM "public"."surface_geometry"
+			WHERE surface_type='ground'
+
+mappingId	BuildingWallSurface
+target		twa:WallSurface/{uuid} obda:isCanonicalIRIOf city:WallSurface/{"id"} .
+            city:WallSurface/{"id"} a bldg:WallSurface . 
+source		SELECT id, uuid
+			FROM "public"."surface_geometry"
+			WHERE surface_type='wall'
+
+mappingId	Surface Geometry to City Object - roof surface
+target		city:RoofSurface/{"cityobject_id"} geo:hasGeometry city:geometry/{"id"} . 
+source		SELECT "id" AS "cityobject_id", "id"
+			FROM "public"."surface_geometry"
+			WHERE "surface_type" = 'roof'
+
+mappingId	Surface Geometry to City Object - ground surface
+target		city:GroundSurface/{"cityobject_id"} geo:hasGeometry city:geometry/{"id"} . 
+source		SELECT "id" AS "cityobject_id", "id"
+			FROM "public"."surface_geometry"
+			WHERE "surface_type" = 'ground'
+
+mappingId	Surface Geometry to City Object - wall surface
+target		city:WallSurface/{"cityobject_id"} geo:hasGeometry city:geometry/{"id"} . 
+source		SELECT "id" AS "cityobject_id", "id"
+			FROM "public"."surface_geometry"
+			WHERE "surface_type" = 'wall'
+
+mappingId	Building bounded by Roof surface
+target		city:Building/{"building_id"} bldg:boundedBy city:RoofSurface/{"thematic_surface_id"} . 
+source		SELECT building_id, id AS thematic_surface_id
+			FROM "public"."surface_geometry"
+			WHERE "surface_type" = 'roof'
+
+mappingId	Building bounded by Ground surface
+target		city:Building/{"building_id"} bldg:boundedBy city:GroundSurface/{"thematic_surface_id"} . 
+source		SELECT building_id, id AS thematic_surface_id
+			FROM "public"."surface_geometry"
+			WHERE "surface_type" = 'ground'
+
+mappingId	Building bounded by Wall surface
+target		city:Building/{"building_id"} bldg:boundedBy city:WallSurface/{"thematic_surface_id"} . 
+source		SELECT building_id, id AS thematic_surface_id
+			FROM "public"."surface_geometry"
+			WHERE "surface_type" = 'wall'
+
+]]

examples/datasets/inputs/data/building-cambridge/vector-toid-uprn.obda

@@ -0,0 +1,21 @@
+[PrefixDeclaration]
+building:   http://theworldavatar.io/BuildingWallSurface/
+os:         http://data.ordnancesurvey.co.uk/ontology/spatialrelations/
+owl:        http://www.w3.org/2002/07/owl#
+rdf:        http://www.w3.org/1999/02/22-rdf-syntax-ns#
+xml:        http://www.w3.org/XML/1998/namespace
+xsd:        http://www.w3.org/2001/XMLSchema#
+obda:       https://w3id.org/obda/vocabulary#
+rdfs:       http://www.w3.org/2000/01/rdf-schema#
+city:		http://cui.unige.ch/citygml/2.0/
+
+[MappingDeclaration] @collection [[
+mappingId       Asset-mapping-TOID-UPRN
+target          city:Building/{"id"} os:hasTOID {os_topo_toid}^^xsd:string ;
+				os:hasUPRN {uprn}^^xsd:integer .
+source          SELECT "public"."building"."id", "public"."building"."os_topo_toid", "public"."TOID_UPRN_MatchingTable"."IDENTIFIER_1" as uprn
+                FROM "public"."building" JOIN "public"."TOID_UPRN_MatchingTable" ON
+				"public"."building"."os_topo_toid" = "public"."TOID_UPRN_MatchingTable"."IDENTIFIER_2"
+
+
+]]

stack-clients/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   stack-client:
-    image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.48.0
+    image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.48.1-XtoCityDB-SNAPSHOT
     secrets:
       - blazegraph_password
       - postgis_password

stack-clients/pom.xml

@@ -7,7 +7,7 @@
 
     <groupId>com.cmclinnovations</groupId>
     <artifactId>stack-clients</artifactId>
-    <version>1.48.0</version>
+    <version>1.48.0-XtoCityDB-SNAPSHOT</version>
 
     <name>Stack Clients</name>
     <url>https://theworldavatar.io</url>

stack-clients/src/main/java/com/cmclinnovations/stack/clients/core/datasets/CityDB.java

@@ -37,6 +37,8 @@ public class CityDB extends GeoServerDataSubset {
     @JsonProperty
     private boolean augmentData = true;
     @JsonProperty
+    private boolean unlogWhenAugment = true;
+    @JsonProperty
     private boolean discoverThematicSurface = false;
     @JsonProperty
     private double critAreaRatio = 0.1;
@@ -109,8 +111,10 @@ protected void loadDataInternal(Path dataSubsetDir, String database, String base
 
     protected void augmentData(String database) {
 
-        logger.info("Setting tables to unlogged for better write performance...");
-        CityDBClient.getInstance().unlogTable(database);
+        if (unlogWhenAugment) {
+            logger.info("Setting tables to unlogged for better write performance...");
+            CityDBClient.getInstance().unlogTable(database);
+        }
 
         if (discoverThematicSurface) {
             logger.info("Discovering thematic surface...");
@@ -121,8 +125,10 @@ protected void augmentData(String database) {
         logger.info("Adding building footprint...");
         CityDBClient.getInstance().addFootprint(database);
 
-        logger.info("Setting tables to logged...");
-        CityDBClient.getInstance().relogTable(database);
+        if (unlogWhenAugment) {
+            logger.info("Setting tables to logged...");
+            CityDBClient.getInstance().relogTable(database);
+        }
     }
 
     @Override