From 57a23e4bd80ed9b492b7ffbd34226b1678a5a13e Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 29 Jan 2026 14:56:47 +0100 Subject: [PATCH 01/57] initial coding size based geometry prioritization --- README.md | 2 +- src/README.md | 0 src/b3dm.tileset/FeatureCountRepository.cs | 7 +- src/b3dm.tileset/GeometryRepository.cs | 35 ++++- src/b3dm.tileset/OctreeTiler.cs | 103 ++++++++++---- src/b3dm.tileset/QuadtreeTiler.cs | 151 +++++++++++++++------ src/b3dm.tileset/SpatialIndexChecker.cs | 2 +- src/pg2b3dm.database.tests/UnitTest1.cs | 8 +- src/pg2b3dm/Program.cs | 2 +- src/wkb2gltf.core/GeometryRecord.cs | 2 + 10 files changed, 230 insertions(+), 82 deletions(-) create mode 100644 src/README.md diff --git a/README.md b/README.md index 2c64bfde..f0f2df2b 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ triangulation. Geometries with interior rings are supported. For large datasets create a spatial index on the geometry column: ``` -psql> CREATE INDEX ON the_table USING gist(st_centroid(st_envelope(geom_triangle))); +psql> CREATE INDEX ON the_table USING gist(st_centroid(st_envelope(geom)), st_envelope(geom)); ``` When there the spatial index is not present the following warning is shown. diff --git a/src/README.md b/src/README.md new file mode 100644 index 00000000..e69de29b diff --git a/src/b3dm.tileset/FeatureCountRepository.cs b/src/b3dm.tileset/FeatureCountRepository.cs index 389a92d7..5e582dd3 100644 --- a/src/b3dm.tileset/FeatureCountRepository.cs +++ b/src/b3dm.tileset/FeatureCountRepository.cs @@ -1,14 +1,15 @@ -using Npgsql; +using System.Collections.Generic; +using Npgsql; using Wkx; namespace B3dm.Tileset; public static class FeatureCountRepository { - public static int CountFeaturesInBox(NpgsqlConnection conn, string geometry_table, string geometry_column, Point from, Point to, string query, int source_epsg, bool keepProjection = false) + public static int CountFeaturesInBox(NpgsqlConnection conn, string geometry_table, string geometry_column, Point from, Point to, string query, int source_epsg, bool keepProjection = false, HashSet excludeHashes = null) { var select = $"COUNT({geometry_column})"; - var where = GeometryRepository.GetWhere(geometry_column, from, to, query, source_epsg, keepProjection); + var where = GeometryRepository.GetWhere(geometry_column, from, to, query, source_epsg, keepProjection, excludeHashes); var sql = $"SELECT {select} FROM {geometry_table} WHERE {where}"; conn.Open(); diff --git a/src/b3dm.tileset/GeometryRepository.cs b/src/b3dm.tileset/GeometryRepository.cs index f9297b46..eeffe65c 100644 --- a/src/b3dm.tileset/GeometryRepository.cs +++ b/src/b3dm.tileset/GeometryRepository.cs @@ -38,7 +38,7 @@ public static double[] GetGeometriesBoundingBox(NpgsqlConnection conn, string ge return result; } - public static List GetGeometrySubset(NpgsqlConnection conn, string geometry_table, string geometry_column, double[] bbox, int source_epsg, int target_srs, string shaderColumn = "", string attributesColumns = "", string query = "", string radiusColumn = "", bool keepProjection = false) + public static List GetGeometrySubset(NpgsqlConnection conn, string geometry_table, string geometry_column, double[] bbox, int source_epsg, int target_srs, string shaderColumn = "", string attributesColumns = "", string query = "", string radiusColumn = "", bool keepProjection = false, HashSet excludeHashes = null) { var sqlselect = GetSqlSelect(geometry_column, shaderColumn, attributesColumns, radiusColumn, target_srs); var sqlFrom = "FROM " + geometry_table; @@ -46,14 +46,15 @@ public static List GetGeometrySubset(NpgsqlConnection conn, stri // todo: fix unit test when there is no z var points = GetPoints(bbox); - var sqlWhere = GetWhere(geometry_column, points.fromPoint, points.toPoint, query, source_epsg, keepProjection); - var sql = sqlselect + sqlFrom + " where " + sqlWhere; + var sqlWhere = GetWhere(geometry_column, points.fromPoint, points.toPoint, query, source_epsg, keepProjection, excludeHashes); + var sqlOrderBy = GetOrderBy(geometry_column); + var sql = sqlselect + sqlFrom + " where " + sqlWhere + sqlOrderBy; - var geometries = GetGeometries(conn, shaderColumn, attributesColumns, sql, radiusColumn); + var geometries = GetGeometries(conn, shaderColumn, attributesColumns, sql, radiusColumn, geometry_column); return geometries; } - public static string GetWhere(string geometry_column, Point from, Point to, string query, int source_epsg, bool keepProjection) + public static string GetWhere(string geometry_column, Point from, Point to, string query, int source_epsg, bool keepProjection, HashSet excludeHashes = null) { var fromX = from.X.Value.ToString(CultureInfo.InvariantCulture); var fromY = from.Y.Value.ToString(CultureInfo.InvariantCulture); @@ -82,6 +83,12 @@ public static string GetWhere(string geometry_column, Point from, Point to, stri $"ST_3DIntersects({geom}, st_transform(ST_3DMakeBox({fromBox}, {toBox}), {source_epsg})) {query}"; } + // Add hash exclusion filter + if (excludeHashes != null && excludeHashes.Count > 0) { + var hashList = string.Join(",", excludeHashes.Select(h => $"'{h}'")); + where += $" AND MD5(ST_AsBinary({geometry_column})::text) NOT IN ({hashList})"; + } + return where; } @@ -98,6 +105,8 @@ public static string GetSqlSelect(string geometry_column, string shaderColumn, s if (radiusColumn != String.Empty) { sqlselect = $"{sqlselect}, {radiusColumn} "; } + // Add MD5 hash of geometry + sqlselect = $"{sqlselect}, MD5(ST_AsBinary({geometry_column})::text) as geom_hash "; return sqlselect; } @@ -107,7 +116,12 @@ public static string GetGeometryColumn(string geometry_column, int target_srs) return $"st_transform({geometry_column}, {target_srs})"; } - public static List GetGeometries(NpgsqlConnection conn, string shaderColumn, string attributesColumns, string sql, string radiusColumn) + public static string GetOrderBy(string geometry_column) + { + return $" ORDER BY ST_Area(ST_Envelope({geometry_column})) DESC"; + } + + public static List GetGeometries(NpgsqlConnection conn, string shaderColumn, string attributesColumns, string sql, string radiusColumn, string geometry_column = "") { var geometries = new List(); conn.Open(); @@ -116,6 +130,7 @@ public static List GetGeometries(NpgsqlConnection conn, string s var attributesColumnIds = new Dictionary(); var shadersColumnId = int.MinValue; var radiusColumnId = int.MinValue; + var hashColumnId = int.MinValue; if (attributesColumns != String.Empty) { var attributesColumnsList = attributesColumns.Split(',').ToList(); @@ -133,6 +148,11 @@ public static List GetGeometries(NpgsqlConnection conn, string s radiusColumnId = FindField(reader, radiusColumn).Value; } } + // Find hash column + var hashFld = FindField(reader, "geom_hash"); + if (hashFld.HasValue) { + hashColumnId = hashFld.Value; + } var batchId = 0; while (reader.Read()) { @@ -154,6 +174,9 @@ public static List GetGeometries(NpgsqlConnection conn, string s var radius = reader.GetFieldValue(radiusColumnId); geometryRecord.Radius = Convert.ToSingle(radius); } + if (hashColumnId != int.MinValue) { + geometryRecord.Hash = reader.GetString(hashColumnId); + } geometries.Add(geometryRecord); batchId++; diff --git a/src/b3dm.tileset/OctreeTiler.cs b/src/b3dm.tileset/OctreeTiler.cs index ac3dd9a8..630503af 100644 --- a/src/b3dm.tileset/OctreeTiler.cs +++ b/src/b3dm.tileset/OctreeTiler.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using B3dm.Tileset.settings; using Npgsql; using pg2b3dm; @@ -28,14 +29,23 @@ public OctreeTiler(string connectionString, InputTable inputTable, TilingSetting public List GenerateTiles3D(BoundingBox3D bbox, int level, Tile3D tile, List tiles) { - return GenerateTiles3D(bbox, level, tile, tiles, null); + return GenerateTiles3D(bbox, level, tile, tiles, null, null); } public List GenerateTiles3D(BoundingBox3D bbox, int level, Tile3D tile, List tiles, Dictionary tileBounds) { + return GenerateTiles3D(bbox, level, tile, tiles, tileBounds, null); + } + + public List GenerateTiles3D(BoundingBox3D bbox, int level, Tile3D tile, List tiles, Dictionary tileBounds, HashSet processedGeometries) + { + if (processedGeometries == null) { + processedGeometries = new HashSet(); + } + var where = inputTable.GetQueryClause(); - var numberOfFeatures = FeatureCountRepository.CountFeaturesInBox(conn, inputTable.TableName, inputTable.GeometryColumn, new Point(bbox.XMin, bbox.YMin, bbox.ZMin), new Point(bbox.XMax, bbox.YMax, bbox.ZMax), where, inputTable.EPSGCode, tilingSettings.KeepProjection); + var numberOfFeatures = FeatureCountRepository.CountFeaturesInBox(conn, inputTable.TableName, inputTable.GeometryColumn, new Point(bbox.XMin, bbox.YMin, bbox.ZMin), new Point(bbox.XMax, bbox.YMax, bbox.ZMax), where, inputTable.EPSGCode, tilingSettings.KeepProjection, processedGeometries); if (numberOfFeatures == 0) { var t2 = new Tile3D(level, tile.X, tile.Y, tile.Z); t2.Available = false; @@ -46,6 +56,9 @@ public List GenerateTiles3D(BoundingBox3D bbox, int level, Tile3D tile, } } else if (numberOfFeatures > tilingSettings.MaxFeaturesPerTile) { + // First, create a tile with the largest geometries up to MaxFeaturesPerTile for this level + CreateTileForLargestGeometries3D(bbox, level, tile, tiles, tileBounds, where, processedGeometries); + level++; for (var x = 0; x < 2; x++) { for (var y = 0; y < 2; y++) { @@ -65,45 +78,85 @@ public List GenerateTiles3D(BoundingBox3D bbox, int level, Tile3D tile, var bbox3d = new BoundingBox3D(xstart, ystart, z_start, xend, yend, zend); var new_tile = new Tile3D(level, tile.X * 2 + x, tile.Y * 2 + y, tile.Z * 2 + z); - GenerateTiles3D(bbox3d, level, new_tile, tiles, tileBounds); + GenerateTiles3D(bbox3d, level, new_tile, tiles, tileBounds, processedGeometries); } } } } else { - var boundingBox = new BoundingBox(bbox.XMin, bbox.YMin, bbox.XMax, bbox.YMax); + CreateTile3D(bbox, level, tile, tiles, tileBounds, where, processedGeometries); + } - int target_srs = 4978; + return tiles; - if (tilingSettings.KeepProjection) { - target_srs = inputTable.EPSGCode; - } + } + + private void CreateTileForLargestGeometries3D(BoundingBox3D bbox, int level, Tile3D tile, List tiles, Dictionary tileBounds, string where, HashSet processedGeometries) + { + // Get the largest geometries (up to MaxFeaturesPerTile) for this tile at this level + int target_srs = tilingSettings.KeepProjection ? inputTable.EPSGCode : 4978; - var bbox1 = new double[] { bbox.XMin, bbox.YMin, bbox.XMax, bbox.YMax, bbox.ZMin, bbox.ZMax }; - var geometries = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, bbox1, inputTable.EPSGCode, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, tilingSettings.KeepProjection); + var bbox1 = new double[] { bbox.XMin, bbox.YMin, bbox.XMax, bbox.YMax, bbox.ZMin, bbox.ZMax }; + var geometries = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, bbox1, inputTable.EPSGCode, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, tilingSettings.KeepProjection, processedGeometries); - if (geometries.Count > 0) { + // Only take up to MaxFeaturesPerTile largest geometries + var geometriesToProcess = geometries.Take(tilingSettings.MaxFeaturesPerTile).ToList(); - if (!tilingSettings.SkipCreateTiles) { - var bytes = TileWriter.ToTile(geometries, tilesetSettings.Translation, copyright: tilesetSettings.Copyright, addOutlines: stylingSettings.AddOutlines, defaultColor: stylingSettings.DefaultColor, defaultMetallicRoughness: stylingSettings.DefaultMetallicRoughness, doubleSided: stylingSettings.DoubleSided, defaultAlphaMode: stylingSettings.DefaultAlphaMode, alphaCutoff: stylingSettings.AlphaCutoff, createGltf: tilingSettings.CreateGltf); - var file = $"{tilesetSettings.OutputSettings.ContentFolder}{Path.AltDirectorySeparatorChar}{tile.Level}_{tile.Z}_{tile.X}_{tile.Y}.glb"; - Console.Write($"\rCreating tile: {file} "); - File.WriteAllBytes($"{file}", bytes); + if (geometriesToProcess.Count > 0) { + // Collect hashes of processed geometries + foreach (var geom in geometriesToProcess) { + if (!string.IsNullOrEmpty(geom.Hash)) { + processedGeometries.Add(geom.Hash); } - tile.Available = true; - } - else { - tile.Available = false; } - tiles.Add(tile); - if (tileBounds != null) { - var key = $"{tile.Level}_{tile.Z}_{tile.X}_{tile.Y}"; - tileBounds[key] = bbox; + if (!tilingSettings.SkipCreateTiles) { + var bytes = TileWriter.ToTile(geometriesToProcess, tilesetSettings.Translation, copyright: tilesetSettings.Copyright, addOutlines: stylingSettings.AddOutlines, defaultColor: stylingSettings.DefaultColor, defaultMetallicRoughness: stylingSettings.DefaultMetallicRoughness, doubleSided: stylingSettings.DoubleSided, defaultAlphaMode: stylingSettings.DefaultAlphaMode, alphaCutoff: stylingSettings.AlphaCutoff, createGltf: tilingSettings.CreateGltf); + var file = $"{tilesetSettings.OutputSettings.ContentFolder}{Path.AltDirectorySeparatorChar}{tile.Level}_{tile.Z}_{tile.X}_{tile.Y}.glb"; + Console.Write($"\rCreating tile: {file} "); + File.WriteAllBytes($"{file}", bytes); } + tile.Available = true; } - return tiles; + tiles.Add(tile); + if (tileBounds != null) { + var key = $"{tile.Level}_{tile.Z}_{tile.X}_{tile.Y}"; + tileBounds[key] = bbox; + } + } + + private void CreateTile3D(BoundingBox3D bbox, int level, Tile3D tile, List tiles, Dictionary tileBounds, string where, HashSet processedGeometries) + { + int target_srs = tilingSettings.KeepProjection ? inputTable.EPSGCode : 4978; + + var bbox1 = new double[] { bbox.XMin, bbox.YMin, bbox.XMax, bbox.YMax, bbox.ZMin, bbox.ZMax }; + var geometries = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, bbox1, inputTable.EPSGCode, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, tilingSettings.KeepProjection, processedGeometries); + if (geometries.Count > 0) { + // Collect hashes of processed geometries + foreach (var geom in geometries) { + if (!string.IsNullOrEmpty(geom.Hash)) { + processedGeometries.Add(geom.Hash); + } + } + + if (!tilingSettings.SkipCreateTiles) { + var bytes = TileWriter.ToTile(geometries, tilesetSettings.Translation, copyright: tilesetSettings.Copyright, addOutlines: stylingSettings.AddOutlines, defaultColor: stylingSettings.DefaultColor, defaultMetallicRoughness: stylingSettings.DefaultMetallicRoughness, doubleSided: stylingSettings.DoubleSided, defaultAlphaMode: stylingSettings.DefaultAlphaMode, alphaCutoff: stylingSettings.AlphaCutoff, createGltf: tilingSettings.CreateGltf); + var file = $"{tilesetSettings.OutputSettings.ContentFolder}{Path.AltDirectorySeparatorChar}{tile.Level}_{tile.Z}_{tile.X}_{tile.Y}.glb"; + Console.Write($"\rCreating tile: {file} "); + File.WriteAllBytes($"{file}", bytes); + } + tile.Available = true; + } + else { + tile.Available = false; + } + + tiles.Add(tile); + if (tileBounds != null) { + var key = $"{tile.Level}_{tile.Z}_{tile.X}_{tile.Y}"; + tileBounds[key] = bbox; + } } } diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index 637f0c95..4f2bd545 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -38,8 +38,12 @@ public QuadtreeTiler(string connectionString, InputTable inputTable, StylingSett this.stylingSettings = stylingSettings; } - public List GenerateTiles(BoundingBox bbox, Tile tile, List tiles, int lod = 0, bool createGltf = false, bool keepProjection = false) + public List GenerateTiles(BoundingBox bbox, Tile tile, List tiles, int lod = 0, bool createGltf = false, bool keepProjection = false, HashSet processedGeometries = null) { + if (processedGeometries == null) { + processedGeometries = new HashSet(); + } + var where = inputTable.GetQueryClause(); var lodquery = LodQuery.GetLodQuery(inputTable.LodColumn, lod); @@ -48,15 +52,15 @@ public List GenerateTiles(BoundingBox bbox, Tile tile, List tiles, i where += $" and {lodquery}"; } - var numberOfFeatures = FeatureCountRepository.CountFeaturesInBox(conn, inputTable.TableName, inputTable.GeometryColumn, new Point(bbox.XMin, bbox.YMin), new Point(bbox.XMax, bbox.YMax), where, source_epsg, keepProjection); + var numberOfFeatures = FeatureCountRepository.CountFeaturesInBox(conn, inputTable.TableName, inputTable.GeometryColumn, new Point(bbox.XMin, bbox.YMin), new Point(bbox.XMax, bbox.YMax), where, source_epsg, keepProjection, processedGeometries); if (numberOfFeatures == 0) { tile.Available = false; tiles.Add(tile); } else if (numberOfFeatures > maxFeaturesPerTile) { - tile.Available = false; - tiles.Add(tile); + // First, get the largest geometries up to maxFeaturesPerTile for this level + CreateTileForLargestGeometries(bbox, tile, tiles, where, lod, createGltf, keepProjection, processedGeometries); var z = tile.Z + 1; @@ -74,11 +78,37 @@ public List GenerateTiles(BoundingBox bbox, Tile tile, List tiles, i var bboxQuad = new BoundingBox(xstart, ystart, xend, yend); var new_tile = new Tile(z, tile.X * 2 + x, tile.Y * 2 + y); new_tile.BoundingBox = bboxQuad.ToArray(); - GenerateTiles(bboxQuad, new_tile, tiles, lod, createGltf, keepProjection); + GenerateTiles(bboxQuad, new_tile, tiles, lod, createGltf, keepProjection, processedGeometries); } } } else { + CreateTile(bbox, tile, tiles, where, lod, createGltf, keepProjection, processedGeometries); + } + + return tiles; + } + + private void CreateTileForLargestGeometries(BoundingBox bbox, Tile tile, List tiles, string where, int lod, bool createGltf, bool keepProjection, HashSet processedGeometries) + { + // Get the largest geometries (up to maxFeaturesPerTile) for this tile at this level + tile.Available = false; + tile.BoundingBox = bbox.ToArray(); + + int target_srs = keepProjection ? source_epsg : 4978; + + var geometries = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, tile.BoundingBox, source_epsg, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, keepProjection, processedGeometries); + + // Only take up to maxFeaturesPerTile largest geometries + var geometriesToProcess = geometries.Take(maxFeaturesPerTile).ToList(); + + if (geometriesToProcess.Count > 0) { + // Collect hashes of processed geometries + foreach (var geom in geometriesToProcess) { + if (!string.IsNullOrEmpty(geom.Hash)) { + processedGeometries.Add(geom.Hash); + } + } var file = $"{tile.Z}_{tile.X}_{tile.Y}"; if (inputTable.LodColumn != String.Empty) { @@ -90,55 +120,94 @@ public List GenerateTiles(BoundingBox bbox, Tile tile, List tiles, i Console.Write($"\rCreating tile: {file} "); tile.ContentUri = file; - int target_srs = 4978; + tile.Lod = lod; - if(keepProjection) { - target_srs = source_epsg; + if (!skipCreateTiles) { + var bytes = TileWriter.ToTile(geometriesToProcess, translation, copyright: copyright, addOutlines: stylingSettings.AddOutlines, defaultColor: stylingSettings.DefaultColor, defaultMetallicRoughness: stylingSettings.DefaultMetallicRoughness, doubleSided: stylingSettings.DoubleSided, defaultAlphaMode: stylingSettings.DefaultAlphaMode, alphaCutoff: stylingSettings.AlphaCutoff, createGltf: createGltf); + File.WriteAllBytes($"{outputFolder}{Path.AltDirectorySeparatorChar}{file}", bytes); } - byte[] bytes = null; + ProcessLodLevels(bbox, tile, lod, createGltf, keepProjection); + UpdateTileBoundingBox(tile, where, keepProjection); - var geometries = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, tile.BoundingBox, source_epsg, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, keepProjection); - // var scale = new double[] { 1, 1, 1 }; - if (geometries.Count > 0) { + tile.Available = true; + } + + tiles.Add(tile); + } - tile.Lod = lod; + private void CreateTile(BoundingBox bbox, Tile tile, List tiles, string where, int lod, bool createGltf, bool keepProjection, HashSet processedGeometries) + { + tile.BoundingBox = bbox.ToArray(); + + var file = $"{tile.Z}_{tile.X}_{tile.Y}"; + if (inputTable.LodColumn != String.Empty) { + file += $"_{lod}"; + } - if (!skipCreateTiles) { - bytes = TileWriter.ToTile(geometries, translation, copyright: copyright, addOutlines: stylingSettings.AddOutlines, defaultColor: stylingSettings.DefaultColor, defaultMetallicRoughness: stylingSettings.DefaultMetallicRoughness, doubleSided: stylingSettings.DoubleSided, defaultAlphaMode: stylingSettings.DefaultAlphaMode, alphaCutoff: stylingSettings.AlphaCutoff, createGltf: createGltf); - File.WriteAllBytes($"{outputFolder}{Path.AltDirectorySeparatorChar}{file}", bytes); - } - if (inputTable.LodColumn != String.Empty) { - if (lod < lods.Max()) { - // take the next lod - var currentIndex = lods.FindIndex(p => p == lod); - var nextIndex = currentIndex + 1; - var nextLod = lods[nextIndex]; - // make a copy of the tile - var t2 = new Tile(tile.Z, tile.X, tile.Y); - t2.BoundingBox = tile.BoundingBox; - var lodNextTiles = GenerateTiles(bbox, t2, new List(), nextLod, createGltf, keepProjection); - tile.Children = lodNextTiles; - }; + var ext = createGltf ? ".glb" : ".b3dm"; + file += ext; + Console.Write($"\rCreating tile: {file} "); + tile.ContentUri = file; + + int target_srs = keepProjection ? source_epsg : 4978; + + var geometries = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, tile.BoundingBox, source_epsg, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, keepProjection, processedGeometries); + + if (geometries.Count > 0) { + // Collect hashes of processed geometries + foreach (var geom in geometries) { + if (!string.IsNullOrEmpty(geom.Hash)) { + processedGeometries.Add(geom.Hash); } + } - // next code is used to fix geometries that have centroid in the tile, but some parts outside... - var bbox_geometries = GeometryRepository.GetGeometriesBoundingBox(conn, inputTable.TableName, inputTable.GeometryColumn, source_epsg, tile, where, keepProjection); - var bbox_tile = new double[] { bbox_geometries[0], bbox_geometries[1], bbox_geometries[2], bbox_geometries[3] }; - tile.BoundingBox = bbox_tile; - tile.ZMin = bbox_geometries[4]; - tile.ZMax = bbox_geometries[5]; + tile.Lod = lod; + + if (!skipCreateTiles) { + var bytes = TileWriter.ToTile(geometries, translation, copyright: copyright, addOutlines: stylingSettings.AddOutlines, defaultColor: stylingSettings.DefaultColor, defaultMetallicRoughness: stylingSettings.DefaultMetallicRoughness, doubleSided: stylingSettings.DoubleSided, defaultAlphaMode: stylingSettings.DefaultAlphaMode, alphaCutoff: stylingSettings.AlphaCutoff, createGltf: createGltf); + File.WriteAllBytes($"{outputFolder}{Path.AltDirectorySeparatorChar}{file}", bytes); + } - tile.Available = true; + ProcessLodLevels(bbox, tile, lod, createGltf, keepProjection); + UpdateTileBoundingBox(tile, where, keepProjection); - if (skipCreateTiles) { tile.Available = true; } + tile.Available = true; + + if (skipCreateTiles) { + tile.Available = true; } - else { - tile.Available = false; + } + else { + tile.Available = false; + } + tiles.Add(tile); + } + + private void ProcessLodLevels(BoundingBox bbox, Tile tile, int lod, bool createGltf, bool keepProjection) + { + if (inputTable.LodColumn != String.Empty) { + if (lod < lods.Max()) { + // take the next lod + var currentIndex = lods.FindIndex(p => p == lod); + var nextIndex = currentIndex + 1; + var nextLod = lods[nextIndex]; + // make a copy of the tile + var t2 = new Tile(tile.Z, tile.X, tile.Y); + t2.BoundingBox = tile.BoundingBox; + var lodNextTiles = GenerateTiles(bbox, t2, new List(), nextLod, createGltf, keepProjection); + tile.Children = lodNextTiles; } - tiles.Add(tile); } + } - return tiles; + private void UpdateTileBoundingBox(Tile tile, string where, bool keepProjection) + { + // next code is used to fix geometries that have centroid in the tile, but some parts outside... + var bbox_geometries = GeometryRepository.GetGeometriesBoundingBox(conn, inputTable.TableName, inputTable.GeometryColumn, source_epsg, tile, where, keepProjection); + var bbox_tile = new double[] { bbox_geometries[0], bbox_geometries[1], bbox_geometries[2], bbox_geometries[3] }; + tile.BoundingBox = bbox_tile; + tile.ZMin = bbox_geometries[4]; + tile.ZMax = bbox_geometries[5]; } } diff --git a/src/b3dm.tileset/SpatialIndexChecker.cs b/src/b3dm.tileset/SpatialIndexChecker.cs index c4d95ae3..496866a1 100644 --- a/src/b3dm.tileset/SpatialIndexChecker.cs +++ b/src/b3dm.tileset/SpatialIndexChecker.cs @@ -20,7 +20,7 @@ public static bool HasSpatialIndex(NpgsqlConnection conn, string geometry_table, cmd.Parameters.AddWithValue("schema", schema); cmd.Parameters.AddWithValue("geometry_table", geometry_table); - cmd.Parameters.AddWithValue("index", "%st_centroid(st_envelope(" + geometry_column + "%))"); + cmd.Parameters.AddWithValue("index", "%st_centroid(st_envelope(" + geometry_column + "%)), st_envelope(" + geometry_column + "%))%"); var reader = cmd.ExecuteReader(); reader.Read(); diff --git a/src/pg2b3dm.database.tests/UnitTest1.cs b/src/pg2b3dm.database.tests/UnitTest1.cs index 39314603..a1b69863 100644 --- a/src/pg2b3dm.database.tests/UnitTest1.cs +++ b/src/pg2b3dm.database.tests/UnitTest1.cs @@ -78,7 +78,7 @@ public void TestArvieuxBuildingsOctree() var implicitTiler = new OctreeTiler(connectionString, inputTable, tilingSettings, stylingSettings, tilesetSettings); var tiles = implicitTiler.GenerateTiles3D(boundingBox3D, 0, new Tile3D(0, 0, 0, 0 ), new List()); - Assert.That(tiles.Count, Is.EqualTo(36)); + Assert.That(tiles.Count, Is.EqualTo(25)); } [Test] @@ -119,7 +119,7 @@ public void TestArvieuxBuildingsOctreeKeepProjection() var implicitTiler = new OctreeTiler(connectionString, inputTable, tilingSettings, stylingSettings, tilesetSettings); var tiles = implicitTiler.GenerateTiles3D(boundingBox3D, 0, new Tile3D(0, 0, 0, 0), new List()); - Assert.That(tiles.Count, Is.EqualTo(36)); + Assert.That(tiles.Count, Is.EqualTo(25)); } @@ -173,7 +173,7 @@ public void ImplicitTilingTest() bbox_wgs84.bbox, new Tile(0, 0, 0), new List()); - Assert.That(tiles.Count, Is.EqualTo(29)); + Assert.That(tiles.Count, Is.EqualTo(17)); } [Test] @@ -206,7 +206,7 @@ public void LodTest() bbox_wgs84.bbox, new Tile(0, 0, 0), new List()); - Assert.That(tiles.Count, Is.EqualTo(145)); + Assert.That(tiles.Count, Is.EqualTo(89)); } diff --git a/src/pg2b3dm/Program.cs b/src/pg2b3dm/Program.cs index 81244c7e..8989d502 100644 --- a/src/pg2b3dm/Program.cs +++ b/src/pg2b3dm/Program.cs @@ -96,7 +96,7 @@ static void Main(string[] args) Console.WriteLine("-----------------------------------------------------------------------------"); Console.WriteLine($"WARNING: No spatial index detected on {inputTable.TableName}.{inputTable.GeometryColumn}"); Console.WriteLine("Fix: add a spatial index, for example: "); - Console.WriteLine($"'CREATE INDEX ON {inputTable.TableName} USING gist(st_centroid(st_envelope({inputTable.GeometryColumn})))'"); + Console.WriteLine($"'CREATE INDEX ON {inputTable.TableName} USING gist(st_centroid(st_envelope({inputTable.GeometryColumn})), st_envelope({inputTable.GeometryColumn}))'"); Console.WriteLine("-----------------------------------------------------------------------------"); Console.WriteLine(); } diff --git a/src/wkb2gltf.core/GeometryRecord.cs b/src/wkb2gltf.core/GeometryRecord.cs index 13c1ba9a..6e3c1617 100644 --- a/src/wkb2gltf.core/GeometryRecord.cs +++ b/src/wkb2gltf.core/GeometryRecord.cs @@ -20,6 +20,8 @@ public GeometryRecord(int batchId) public float? Radius { get; set; } + public string Hash { get; set; } + public List GetTriangles(double[] translation = null, double[] scale = null) { var triangles = GeometryProcessor.GetTriangles(Geometry, BatchId, translation, scale, Shader, Radius); From 44437114a4fd6ecd1094c07ef5bda85210bcd153 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 29 Jan 2026 15:17:13 +0100 Subject: [PATCH 02/57] limit geometries to retrieve --- src/b3dm.tileset/GeometryRepository.cs | 5 +++-- src/b3dm.tileset/OctreeTiler.cs | 5 +---- src/b3dm.tileset/QuadtreeTiler.cs | 5 +---- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/b3dm.tileset/GeometryRepository.cs b/src/b3dm.tileset/GeometryRepository.cs index eeffe65c..bc48098f 100644 --- a/src/b3dm.tileset/GeometryRepository.cs +++ b/src/b3dm.tileset/GeometryRepository.cs @@ -38,7 +38,7 @@ public static double[] GetGeometriesBoundingBox(NpgsqlConnection conn, string ge return result; } - public static List GetGeometrySubset(NpgsqlConnection conn, string geometry_table, string geometry_column, double[] bbox, int source_epsg, int target_srs, string shaderColumn = "", string attributesColumns = "", string query = "", string radiusColumn = "", bool keepProjection = false, HashSet excludeHashes = null) + public static List GetGeometrySubset(NpgsqlConnection conn, string geometry_table, string geometry_column, double[] bbox, int source_epsg, int target_srs, string shaderColumn = "", string attributesColumns = "", string query = "", string radiusColumn = "", bool keepProjection = false, HashSet excludeHashes = null, int? maxFeatures = null) { var sqlselect = GetSqlSelect(geometry_column, shaderColumn, attributesColumns, radiusColumn, target_srs); var sqlFrom = "FROM " + geometry_table; @@ -48,7 +48,8 @@ public static List GetGeometrySubset(NpgsqlConnection conn, stri var sqlWhere = GetWhere(geometry_column, points.fromPoint, points.toPoint, query, source_epsg, keepProjection, excludeHashes); var sqlOrderBy = GetOrderBy(geometry_column); - var sql = sqlselect + sqlFrom + " where " + sqlWhere + sqlOrderBy; + var sqlLimit = maxFeatures.HasValue ? $" LIMIT {maxFeatures.Value}" : ""; + var sql = sqlselect + sqlFrom + " where " + sqlWhere + sqlOrderBy + sqlLimit; var geometries = GetGeometries(conn, shaderColumn, attributesColumns, sql, radiusColumn, geometry_column); return geometries; diff --git a/src/b3dm.tileset/OctreeTiler.cs b/src/b3dm.tileset/OctreeTiler.cs index 630503af..edcc6e9a 100644 --- a/src/b3dm.tileset/OctreeTiler.cs +++ b/src/b3dm.tileset/OctreeTiler.cs @@ -97,10 +97,7 @@ private void CreateTileForLargestGeometries3D(BoundingBox3D bbox, int level, Til int target_srs = tilingSettings.KeepProjection ? inputTable.EPSGCode : 4978; var bbox1 = new double[] { bbox.XMin, bbox.YMin, bbox.XMax, bbox.YMax, bbox.ZMin, bbox.ZMax }; - var geometries = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, bbox1, inputTable.EPSGCode, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, tilingSettings.KeepProjection, processedGeometries); - - // Only take up to MaxFeaturesPerTile largest geometries - var geometriesToProcess = geometries.Take(tilingSettings.MaxFeaturesPerTile).ToList(); + var geometriesToProcess = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, bbox1, inputTable.EPSGCode, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, tilingSettings.KeepProjection, processedGeometries, tilingSettings.MaxFeaturesPerTile); if (geometriesToProcess.Count > 0) { // Collect hashes of processed geometries diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index 4f2bd545..3c92c12f 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -97,10 +97,7 @@ private void CreateTileForLargestGeometries(BoundingBox bbox, Tile tile, List 0) { // Collect hashes of processed geometries From 3ce5544b0a294b3e8516b8469d31e978be090c7b Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Tue, 3 Feb 2026 12:57:17 +0100 Subject: [PATCH 03/57] improve performance --- src/b3dm.tileset/GeometryRepository.cs | 7 +++-- src/b3dm.tileset/QuadtreeTiler.cs | 34 +++++++++++++++++-------- src/pg2b3dm.database.tests/UnitTest1.cs | 9 ++++--- src/pg2b3dm/Program.cs | 2 +- 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/src/b3dm.tileset/GeometryRepository.cs b/src/b3dm.tileset/GeometryRepository.cs index bc48098f..93c65488 100644 --- a/src/b3dm.tileset/GeometryRepository.cs +++ b/src/b3dm.tileset/GeometryRepository.cs @@ -16,12 +16,15 @@ public static class GeometryRepository /// /// Returns double array with 6 bounding box coordinates, xmin, ymin, xmax, ymax, zmin, zmax /// - public static double[] GetGeometriesBoundingBox(NpgsqlConnection conn, string geometry_table, string geometry_column, int epsg, Tile t, string query = "", bool keepProjection = false) + public static double[] GetGeometriesBoundingBox(NpgsqlConnection conn, string geometry_table, string geometry_column, int epsg, Tile t, HashSet tileHashes, string query = "", bool keepProjection = false) { var sqlSelect = keepProjection? $"select st_Asbinary(st_3dextent({geometry_column})) ": $"select st_Asbinary(st_3dextent(st_transform({geometry_column}, 4979))) "; - var sqlWhere = GetWhere(geometry_column, new Point(t.BoundingBox[0], t.BoundingBox[1]), new Point(t.BoundingBox[2], t.BoundingBox[3]), query, epsg, keepProjection); + + var hashList = string.Join(",", tileHashes.Select(h => $"'{h}'")); + + var sqlWhere = $" MD5(ST_AsBinary({geometry_column})::text) in ({hashList})"; var sql = $"{sqlSelect} from {geometry_table} where {sqlWhere}"; conn.Open(); diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index 3c92c12f..70412ad2 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -23,8 +23,9 @@ public class QuadtreeTiler private readonly bool skipCreateTiles; private readonly StylingSettings stylingSettings; private InputTable inputTable; + private bool useImplicitTiling = false; - public QuadtreeTiler(string connectionString, InputTable inputTable, StylingSettings stylingSettings, int maxFeaturesPerTile, double[] translation, string outputFolder, List lods, string copyright = "", bool skipCreateTiles = false) + public QuadtreeTiler(string connectionString, InputTable inputTable, StylingSettings stylingSettings, int maxFeaturesPerTile, double[] translation, string outputFolder, List lods, string copyright = "", bool skipCreateTiles = false, bool useImplicitTiling=false) { this.conn = new NpgsqlConnection(connectionString); this.inputTable = inputTable; @@ -36,6 +37,7 @@ public QuadtreeTiler(string connectionString, InputTable inputTable, StylingSett this.copyright = copyright; this.skipCreateTiles = skipCreateTiles; this.stylingSettings = stylingSettings; + this.useImplicitTiling = useImplicitTiling; } public List GenerateTiles(BoundingBox bbox, Tile tile, List tiles, int lod = 0, bool createGltf = false, bool keepProjection = false, HashSet processedGeometries = null) @@ -60,7 +62,7 @@ public List GenerateTiles(BoundingBox bbox, Tile tile, List tiles, i } else if (numberOfFeatures > maxFeaturesPerTile) { // First, get the largest geometries up to maxFeaturesPerTile for this level - CreateTileForLargestGeometries(bbox, tile, tiles, where, lod, createGltf, keepProjection, processedGeometries); + var localProcessedGeometries = CreateTileForLargestGeometries(bbox, tile, tiles, where, lod, createGltf, keepProjection, processedGeometries); var z = tile.Z + 1; @@ -78,7 +80,7 @@ public List GenerateTiles(BoundingBox bbox, Tile tile, List tiles, i var bboxQuad = new BoundingBox(xstart, ystart, xend, yend); var new_tile = new Tile(z, tile.X * 2 + x, tile.Y * 2 + y); new_tile.BoundingBox = bboxQuad.ToArray(); - GenerateTiles(bboxQuad, new_tile, tiles, lod, createGltf, keepProjection, processedGeometries); + GenerateTiles(bboxQuad, new_tile, tiles, lod, createGltf, keepProjection, localProcessedGeometries); } } } @@ -89,8 +91,12 @@ public List GenerateTiles(BoundingBox bbox, Tile tile, List tiles, i return tiles; } - private void CreateTileForLargestGeometries(BoundingBox bbox, Tile tile, List tiles, string where, int lod, bool createGltf, bool keepProjection, HashSet processedGeometries) + private HashSet CreateTileForLargestGeometries(BoundingBox bbox, Tile tile, List tiles, string where, int lod, bool createGltf, bool keepProjection, HashSet processedGeometries) { + // clone processedIds to avoid modifying the original set in recursive calls + var localProcessedGeometries = new HashSet(processedGeometries); + var tileHashes = new HashSet(); + // Get the largest geometries (up to maxFeaturesPerTile) for this tile at this level tile.Available = false; tile.BoundingBox = bbox.ToArray(); @@ -100,10 +106,12 @@ private void CreateTileForLargestGeometries(BoundingBox bbox, Tile tile, List 0) { + // Collect hashes of processed geometries foreach (var geom in geometriesToProcess) { if (!string.IsNullOrEmpty(geom.Hash)) { - processedGeometries.Add(geom.Hash); + localProcessedGeometries.Add(geom.Hash); + tileHashes.Add(geom.Hash); } } @@ -125,18 +133,23 @@ private void CreateTileForLargestGeometries(BoundingBox bbox, Tile tile, List tiles, string where, int lod, bool createGltf, bool keepProjection, HashSet processedGeometries) { tile.BoundingBox = bbox.ToArray(); - + var tileHashes = new HashSet(); + var file = $"{tile.Z}_{tile.X}_{tile.Y}"; if (inputTable.LodColumn != String.Empty) { file += $"_{lod}"; @@ -155,6 +168,7 @@ private void CreateTile(BoundingBox bbox, Tile tile, List tiles, string wh // Collect hashes of processed geometries foreach (var geom in geometries) { if (!string.IsNullOrEmpty(geom.Hash)) { + tileHashes.Add(geom.Hash); processedGeometries.Add(geom.Hash); } } @@ -167,7 +181,7 @@ private void CreateTile(BoundingBox bbox, Tile tile, List tiles, string wh } ProcessLodLevels(bbox, tile, lod, createGltf, keepProjection); - UpdateTileBoundingBox(tile, where, keepProjection); + UpdateTileBoundingBox(tile, tileHashes, where, keepProjection); tile.Available = true; @@ -198,10 +212,10 @@ private void ProcessLodLevels(BoundingBox bbox, Tile tile, int lod, bool createG } } - private void UpdateTileBoundingBox(Tile tile, string where, bool keepProjection) + private void UpdateTileBoundingBox(Tile tile, HashSet tileHashes, string where, bool keepProjection) { // next code is used to fix geometries that have centroid in the tile, but some parts outside... - var bbox_geometries = GeometryRepository.GetGeometriesBoundingBox(conn, inputTable.TableName, inputTable.GeometryColumn, source_epsg, tile, where, keepProjection); + var bbox_geometries = GeometryRepository.GetGeometriesBoundingBox(conn, inputTable.TableName, inputTable.GeometryColumn, source_epsg, tile, tileHashes, where, keepProjection); var bbox_tile = new double[] { bbox_geometries[0], bbox_geometries[1], bbox_geometries[2], bbox_geometries[3] }; tile.BoundingBox = bbox_tile; tile.ZMin = bbox_geometries[4]; diff --git a/src/pg2b3dm.database.tests/UnitTest1.cs b/src/pg2b3dm.database.tests/UnitTest1.cs index a1b69863..789a74f7 100644 --- a/src/pg2b3dm.database.tests/UnitTest1.cs +++ b/src/pg2b3dm.database.tests/UnitTest1.cs @@ -168,7 +168,8 @@ public void ImplicitTilingTest() trans, "output/content", new List() { 0 }, - skipCreateTiles: true); + skipCreateTiles: true, + useImplicitTiling: true); var tiles = implicitTiler.GenerateTiles( bbox_wgs84.bbox, new Tile(0, 0, 0), @@ -201,7 +202,8 @@ public void LodTest() trans, "output/content", new List() { 0, 1 }, - skipCreateTiles: true); + skipCreateTiles: true, + useImplicitTiling: true); var tiles = implicitTiler.GenerateTiles( bbox_wgs84.bbox, new Tile(0, 0, 0), @@ -231,7 +233,8 @@ public void GeometryTest() trans, "output/content", new List() { 0 }, - skipCreateTiles: false); + skipCreateTiles: false, + useImplicitTiling: true); var tile = new Tile(0, 0, 0) { BoundingBox = bbox_wgs84.bbox.ToArray() }; diff --git a/src/pg2b3dm/Program.cs b/src/pg2b3dm/Program.cs index 8989d502..cd5f7d6e 100644 --- a/src/pg2b3dm/Program.cs +++ b/src/pg2b3dm/Program.cs @@ -278,7 +278,7 @@ private static void QuadtreeTile(string connectionString, InputTable inputTable, tile.BoundingBox = bbox.ToArray(); var outputSettings = tilesetSettings.OutputSettings; - var quadtreeTiler = new QuadtreeTiler(connectionString, inputTable, stylingSettings, tilingSettings.MaxFeaturesPerTile, tilesetSettings.Translation, outputSettings.ContentFolder, tilingSettings.Lods, tilesetSettings.Copyright, tilingSettings.SkipCreateTiles); + var quadtreeTiler = new QuadtreeTiler(connectionString, inputTable, stylingSettings, tilingSettings.MaxFeaturesPerTile, tilesetSettings.Translation, outputSettings.ContentFolder, tilingSettings.Lods, tilesetSettings.Copyright, tilingSettings.SkipCreateTiles, tilingSettings.UseImplicitTiling); var tiles = quadtreeTiler.GenerateTiles(bbox, tile, new List(), inputTable.LodColumn != string.Empty ? tilingSettings.Lods.First() : 0, tilingSettings.CreateGltf, tilingSettings.KeepProjection); Console.WriteLine(); Console.WriteLine("Tiles created: " + tiles.Count(tile => tile.Available)); From 6cc9ebce0eb31efef95498a2e02d929e1b62f539 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Tue, 3 Feb 2026 13:44:04 +0100 Subject: [PATCH 04/57] add md5 queries --- src/md5_queries.md | 55 ++++++++++++++++++++++++++++++++++++++++++++++ src/pg2b3dm.sln | 1 + 2 files changed, 56 insertions(+) create mode 100644 src/md5_queries.md diff --git a/src/md5_queries.md b/src/md5_queries.md new file mode 100644 index 00000000..a64dd9ed --- /dev/null +++ b/src/md5_queries.md @@ -0,0 +1,55 @@ +## Queries for MD5 + + +### Initial + +1] Get bounding box whole table (1.9 s) + +```sql +SELECT st_xmin(geom1),st_ymin(geom1), st_xmax(geom1), st_ymax(geom1), st_zmin(geom1), st_zmax(geom1) FROM (select st_transform(ST_3DExtent(geom), 4979) as geom1 from bertt.nantes_reconstructed_buildings ) as t +``` +Result: + +``` +-1.8471041030488762 47.14626298148698 -1.1473131952502678 47.62268076404559 34.427586472817715 475.03764899302183 +``` + +## Tile 0_0_0.glb + +2] Count geometries in bounding box (0.2s) + +```sql +SELECT COUNT(geom) FROM bertt.nantes_reconstructed_buildings WHERE ST_Centroid(ST_Envelope(geom)) && st_transform(ST_MakeEnvelope(-1.847105103048876, 47.14626198148698, -1.1473121952502678, 47.62268176404559, 4326), 5698) +``` + +Result: 385856 + +3] Get geometries for tile 0_0_0.glb - 1000 largest geometries in whole table (2 s) + +```sql +SELECT ST_AsBinary(st_transform(geom, 4978)), id , MD5(ST_AsBinary(geom)::text) as geom_hash FROM bertt.nantes_reconstructed_buildings where ST_Centroid(ST_Envelope(geom)) && st_transform(ST_MakeEnvelope(-1.847105103048876, 47.14626198148698, -1.1473121952502678, 47.62268176404559, 4326), 5698) ORDER BY ST_Area(ST_Envelope(geom)) DESC LIMIT 1000 +``` + +md5 hashes (for example '9759cdee666f512a0c13df8245b667f9') are remembered to be excluded in higher level (z) tile + +## Tile 1_0_0.glb (level 1, x=0, y=0) + +4] Count geometries in bounding box on level 1 excluding 1000 largest geometries from tile 0_0_0.glb (8 seconds!) + +```sql +"SELECT COUNT(geom) FROM bertt.nantes_reconstructed_buildings WHERE ST_Centroid(ST_Envelope(geom)) && st_transform(ST_MakeEnvelope(-1.847105103048876, 47.14626198148698, -1.497208649149572, 47.384471872766284, 4326), 5698) AND MD5(ST_AsBinary(geom)::text) NOT IN ('9759cdee666f512a0c13df8245b667f9',..1000 items, ...) +``` + +Result: 235787 + +5] Get geometries for tile 1_0_0.glb - 1000 largest geometries in tile 1_0_0 (10 seconds!) + +```sql +SELECT ST_AsBinary(st_transform(geom, 4978)), id , MD5(ST_AsBinary(geom)::text) as geom_hash FROM bertt.nantes_reconstructed_buildings where ST_Centroid(ST_Envelope(geom)) && st_transform(ST_MakeEnvelope(-1.847105103048876, 47.14626198148698, -1.497208649149572, 47.384471872766284, 4326), 5698) AND MD5(ST_AsBinary(geom)::text) NOT IN ('9759cdee666f512a0c13df8245b667f9', ..1000 items, ...) ORDER BY ST_Area(ST_Envelope(geom)) DESC LIMIT 1000 +``` + +Todo: + +- Check spatial indexes + +- limit 'not in' list of md5 hashes from above level, e.g. only use the hashes of the geometries that intersect the tile envelope diff --git a/src/pg2b3dm.sln b/src/pg2b3dm.sln index 63549f8b..31bec38b 100644 --- a/src/pg2b3dm.sln +++ b/src/pg2b3dm.sln @@ -25,6 +25,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "doc", "doc", "{0403ED2E-B5A ..\..\..\..\..\Users\bert\Desktop\demo_pg2b3dm.gif = ..\..\..\..\..\Users\bert\Desktop\demo_pg2b3dm.gif ..\getting_started.md = ..\getting_started.md C:\Users\bert\Desktop\test.csv\materials_for_features.csv = C:\Users\bert\Desktop\test.csv\materials_for_features.csv + md5_queries.md = md5_queries.md ..\README.md = ..\README.md release_notes_0.10.md = release_notes_0.10.md release_notes_0.11.md = release_notes_0.11.md From 8c0d428cf92558f3c4b2613fc15470b7b444a1ed Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Tue, 3 Feb 2026 13:46:17 +0100 Subject: [PATCH 05/57] move md5 queries --- src/md5_queries.md => md5_queries.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/md5_queries.md => md5_queries.md (100%) diff --git a/src/md5_queries.md b/md5_queries.md similarity index 100% rename from src/md5_queries.md rename to md5_queries.md From c164c8aa68cc252785442e68f3697464ed0c8008 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Tue, 3 Feb 2026 13:49:05 +0100 Subject: [PATCH 06/57] Update md5_queries.md --- md5_queries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/md5_queries.md b/md5_queries.md index a64dd9ed..3433adc5 100644 --- a/md5_queries.md +++ b/md5_queries.md @@ -37,7 +37,7 @@ md5 hashes (for example '9759cdee666f512a0c13df8245b667f9') are remembered to be 4] Count geometries in bounding box on level 1 excluding 1000 largest geometries from tile 0_0_0.glb (8 seconds!) ```sql -"SELECT COUNT(geom) FROM bertt.nantes_reconstructed_buildings WHERE ST_Centroid(ST_Envelope(geom)) && st_transform(ST_MakeEnvelope(-1.847105103048876, 47.14626198148698, -1.497208649149572, 47.384471872766284, 4326), 5698) AND MD5(ST_AsBinary(geom)::text) NOT IN ('9759cdee666f512a0c13df8245b667f9',..1000 items, ...) +SELECT COUNT(geom) FROM bertt.nantes_reconstructed_buildings WHERE ST_Centroid(ST_Envelope(geom)) && st_transform(ST_MakeEnvelope(-1.847105103048876, 47.14626198148698, -1.497208649149572, 47.384471872766284, 4326), 5698) AND MD5(ST_AsBinary(geom)::text) NOT IN ('9759cdee666f512a0c13df8245b667f9',..1000 items, ...) ``` Result: 235787 From 8fff1f71c7887c752bb0a3f5b2ba62cbaf9d10c6 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Tue, 3 Feb 2026 14:12:08 +0100 Subject: [PATCH 07/57] Update md5_queries.md --- md5_queries.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/md5_queries.md b/md5_queries.md index 3433adc5..1dfbdcb5 100644 --- a/md5_queries.md +++ b/md5_queries.md @@ -53,3 +53,5 @@ Todo: - Check spatial indexes - limit 'not in' list of md5 hashes from above level, e.g. only use the hashes of the geometries that intersect the tile envelope + +- idea: make a temporary blacklist table with the to be exluded hashes? From 5415785388f5eb5193a12208d001a6f04b57e8e0 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Tue, 3 Feb 2026 14:13:44 +0100 Subject: [PATCH 08/57] Enhance md5_queries.md with issue and todo sections Added issue section to address slow queries due to long hash lists and provided todo items for optimization. --- md5_queries.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/md5_queries.md b/md5_queries.md index 1dfbdcb5..0f4cc0a6 100644 --- a/md5_queries.md +++ b/md5_queries.md @@ -48,7 +48,11 @@ Result: 235787 SELECT ST_AsBinary(st_transform(geom, 4978)), id , MD5(ST_AsBinary(geom)::text) as geom_hash FROM bertt.nantes_reconstructed_buildings where ST_Centroid(ST_Envelope(geom)) && st_transform(ST_MakeEnvelope(-1.847105103048876, 47.14626198148698, -1.497208649149572, 47.384471872766284, 4326), 5698) AND MD5(ST_AsBinary(geom)::text) NOT IN ('9759cdee666f512a0c13df8245b667f9', ..1000 items, ...) ORDER BY ST_Area(ST_Envelope(geom)) DESC LIMIT 1000 ``` -Todo: +## Issue + +List of hashes can get long (maximum (z+1)*1000 items), giving more slow query + +## Todo - Check spatial indexes From 05ab3639cf995ee1cb6e02c0c5d1c4c4a2a5520b Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Tue, 3 Feb 2026 14:15:47 +0100 Subject: [PATCH 09/57] Update md5_queries.md --- md5_queries.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/md5_queries.md b/md5_queries.md index 0f4cc0a6..7dcf34b5 100644 --- a/md5_queries.md +++ b/md5_queries.md @@ -56,6 +56,11 @@ List of hashes can get long (maximum (z+1)*1000 items), giving more slow query - Check spatial indexes -- limit 'not in' list of md5 hashes from above level, e.g. only use the hashes of the geometries that intersect the tile envelope +- limit 'not in' list of md5 hashes from above level, e.g. only use the hashes of the geometries that intersect the tile envelope. +Will probably help a bit but not enough (list of hashes stays long) - idea: make a temporary blacklist table with the to be exluded hashes? + +- idea: force use of id column (longs)? + +- Other solutions? From d0debce89abe96f415d76d24ecc3d0ce0c0ddb30 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Tue, 3 Feb 2026 14:16:31 +0100 Subject: [PATCH 10/57] Fix formatting issue in md5_queries.md --- md5_queries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/md5_queries.md b/md5_queries.md index 7dcf34b5..646c6038 100644 --- a/md5_queries.md +++ b/md5_queries.md @@ -50,7 +50,7 @@ SELECT ST_AsBinary(st_transform(geom, 4978)), id , MD5(ST_AsBinary(geom)::text) ## Issue -List of hashes can get long (maximum (z+1)*1000 items), giving more slow query +List of hashes can get long (maximum (z*1000 items), giving more slow query ## Todo From 28a9333559473868f852f7a376299705a9f3872b Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Tue, 3 Feb 2026 14:19:34 +0100 Subject: [PATCH 11/57] Propose exception for first tile on z=0 Added potential improvement for z=0 tile handling in md5_queries. --- md5_queries.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/md5_queries.md b/md5_queries.md index 646c6038..d88aca52 100644 --- a/md5_queries.md +++ b/md5_queries.md @@ -32,6 +32,8 @@ SELECT ST_AsBinary(st_transform(geom, 4978)), id , MD5(ST_AsBinary(geom)::text) md5 hashes (for example '9759cdee666f512a0c13df8245b667f9') are remembered to be excluded in higher level (z) tile +potential improvement: make exception for first tile on z=0 - do not filter on envelope (all features are included) + ## Tile 1_0_0.glb (level 1, x=0, y=0) 4] Count geometries in bounding box on level 1 excluding 1000 largest geometries from tile 0_0_0.glb (8 seconds!) From ec867b39fbde835cb9ee10d66f4e7e6ea4740e88 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Wed, 4 Feb 2026 14:09:18 +0100 Subject: [PATCH 12/57] filter where hashes based on tile envelope --- src/b3dm.tileset/GeometryRepository.cs | 39 ++++++++++++++++++++++++++ src/b3dm.tileset/QuadtreeTiler.cs | 5 +++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/b3dm.tileset/GeometryRepository.cs b/src/b3dm.tileset/GeometryRepository.cs index 93c65488..a898e0ff 100644 --- a/src/b3dm.tileset/GeometryRepository.cs +++ b/src/b3dm.tileset/GeometryRepository.cs @@ -13,6 +13,45 @@ namespace B3dm.Tileset; public static class GeometryRepository { + public static HashSet FilterHashesByEnvelope(NpgsqlConnection conn, string tableName, string geometryColumn, BoundingBox bbox, int source_epsg, HashSet geometryHashes, bool keepProjection) + { + if (geometryHashes.Count == 0) { + return new HashSet(); + } + + var filteredHashes = new HashSet(); + + var hashList = string.Join(",", geometryHashes.Select(h => $"'{h}'")); + + conn.Open(); + + var query = $@" + SELECT MD5(ST_AsBinary({geometryColumn})::text) as geom_hash + FROM {tableName} + WHERE MD5(ST_AsBinary({geometryColumn})::text) in ({hashList}) + AND ST_Within( + ST_Centroid(ST_Envelope({geometryColumn})), + ST_Transform(ST_MakeEnvelope(@xmin, @ymin, @xmax, @ymax, 4326), {source_epsg}) + )"; + + using var cmd = new NpgsqlCommand(query, conn); + cmd.Parameters.AddWithValue("hashes", geometryHashes.ToArray()); + cmd.Parameters.AddWithValue("xmin", bbox.XMin); + cmd.Parameters.AddWithValue("ymin", bbox.YMin); + cmd.Parameters.AddWithValue("xmax", bbox.XMax); + cmd.Parameters.AddWithValue("ymax", bbox.YMax); + + using var reader = cmd.ExecuteReader(); + while (reader.Read()) { + var hash = reader.GetString(0); + filteredHashes.Add(hash); + } + + conn.Close(); + + return filteredHashes; + } + /// /// Returns double array with 6 bounding box coordinates, xmin, ymin, xmax, ymax, zmin, zmax /// diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index 70412ad2..af56fd6e 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -80,7 +80,10 @@ public List GenerateTiles(BoundingBox bbox, Tile tile, List tiles, i var bboxQuad = new BoundingBox(xstart, ystart, xend, yend); var new_tile = new Tile(z, tile.X * 2 + x, tile.Y * 2 + y); new_tile.BoundingBox = bboxQuad.ToArray(); - GenerateTiles(bboxQuad, new_tile, tiles, lod, createGltf, keepProjection, localProcessedGeometries); + + var filteredProcessedGeometries = GeometryRepository.FilterHashesByEnvelope(conn, inputTable.TableName, inputTable.GeometryColumn, bboxQuad, source_epsg, localProcessedGeometries, keepProjection); + + GenerateTiles(bboxQuad, new_tile, tiles, lod, createGltf, keepProjection, filteredProcessedGeometries); } } } From c99b726f255904cd18184c21b782022e896807d0 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Wed, 4 Feb 2026 15:08:39 +0100 Subject: [PATCH 13/57] Document spatial indexing recommendations for MD5 hashes Added recommended spatial indexing strategies for MD5 hashes, including SQL commands and performance notes. --- md5_queries.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/md5_queries.md b/md5_queries.md index d88aca52..37bb2f21 100644 --- a/md5_queries.md +++ b/md5_queries.md @@ -54,6 +54,41 @@ SELECT ST_AsBinary(st_transform(geom, 4978)), id , MD5(ST_AsBinary(geom)::text) List of hashes can get long (maximum (z*1000 items), giving more slow query +## Spatial indexing + + Recommended Indexes + + 1. Spatial Index with MD5 Hash (Composite) + +``` + CREATE INDEX idx_geom_centroid_hash ON the_table + USING btree(MD5(ST_AsBinary(geom_triangle)::text)); +``` + 2. Spatial Index (GIST) - Still Required + +``` + CREATE INDEX idx_geom_centroid_spatial ON the_table + USING gist(ST_Centroid(ST_Envelope(geom_triangle))); +``` + + Rationale + + The queries now use three main patterns: + + - Spatial filtering with MD5 hash exclusion (GetGeometrySubset): WHERE ST_Centroid(ST_Envelope(geom_triangle)) && + AND MD5(ST_AsBinary(geom_triangle)::text) NOT IN () + - MD5 hash filtering with spatial validation (FilterHashesByEnvelope): WHERE MD5(ST_AsBinary(geom_triangle)::text) IN () + AND ST_Within(ST_Centroid(ST_Envelope(geom_triangle)), ) + - Hash-only filtering (GetGeometriesBoundingBox): WHERE MD5(ST_AsBinary(geom_triangle)::text) IN () + + Performance Notes: + + - The GIST spatial index handles the ST_Centroid(ST_Envelope(geom_triangle)) predicates + - The MD5 hash BTREE index handles the MD5(ST_AsBinary(geom_triangle)::text) IN/NOT IN predicates + - PostgreSQL will use both indexes (bitmap index scan) for queries with both predicates + + Optional: Materialized Hash Column + ## Todo - Check spatial indexes From 4137dac2f88fe8effa1b9d0851e6176513ec8443 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Wed, 4 Feb 2026 15:12:18 +0100 Subject: [PATCH 14/57] Fix header formatting and clean up index creation examples Corrected header formatting and removed unnecessary code blocks for index creation. --- md5_queries.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/md5_queries.md b/md5_queries.md index 37bb2f21..659f7e33 100644 --- a/md5_queries.md +++ b/md5_queries.md @@ -1,4 +1,4 @@ -## Queries for MD5 +giv## Queries for MD5 ### Initial @@ -60,16 +60,13 @@ List of hashes can get long (maximum (z*1000 items), giving more slow query 1. Spatial Index with MD5 Hash (Composite) -``` CREATE INDEX idx_geom_centroid_hash ON the_table USING btree(MD5(ST_AsBinary(geom_triangle)::text)); -``` + 2. Spatial Index (GIST) - Still Required -``` CREATE INDEX idx_geom_centroid_spatial ON the_table USING gist(ST_Centroid(ST_Envelope(geom_triangle))); -``` Rationale From 0b5ae57e0fa3b727f39d9fb179253960cd520996 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Wed, 4 Feb 2026 15:13:13 +0100 Subject: [PATCH 15/57] Numbered list formatting for query patterns --- md5_queries.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/md5_queries.md b/md5_queries.md index 659f7e33..63b1a60b 100644 --- a/md5_queries.md +++ b/md5_queries.md @@ -72,17 +72,17 @@ List of hashes can get long (maximum (z*1000 items), giving more slow query The queries now use three main patterns: - - Spatial filtering with MD5 hash exclusion (GetGeometrySubset): WHERE ST_Centroid(ST_Envelope(geom_triangle)) && + 1] Spatial filtering with MD5 hash exclusion (GetGeometrySubset): WHERE ST_Centroid(ST_Envelope(geom_triangle)) && AND MD5(ST_AsBinary(geom_triangle)::text) NOT IN () - - MD5 hash filtering with spatial validation (FilterHashesByEnvelope): WHERE MD5(ST_AsBinary(geom_triangle)::text) IN () + 2] MD5 hash filtering with spatial validation (FilterHashesByEnvelope): WHERE MD5(ST_AsBinary(geom_triangle)::text) IN () AND ST_Within(ST_Centroid(ST_Envelope(geom_triangle)), ) - - Hash-only filtering (GetGeometriesBoundingBox): WHERE MD5(ST_AsBinary(geom_triangle)::text) IN () + 3] Hash-only filtering (GetGeometriesBoundingBox): WHERE MD5(ST_AsBinary(geom_triangle)::text) IN () Performance Notes: - - The GIST spatial index handles the ST_Centroid(ST_Envelope(geom_triangle)) predicates - - The MD5 hash BTREE index handles the MD5(ST_AsBinary(geom_triangle)::text) IN/NOT IN predicates - - PostgreSQL will use both indexes (bitmap index scan) for queries with both predicates + 1] The GIST spatial index handles the ST_Centroid(ST_Envelope(geom_triangle)) predicates + 2] The MD5 hash BTREE index handles the MD5(ST_AsBinary(geom_triangle)::text) IN/NOT IN predicates + 3] PostgreSQL will use both indexes (bitmap index scan) for queries with both predicates Optional: Materialized Hash Column From 05d00bb7c70d0b053d5c107104ae31db0f738799 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Wed, 4 Feb 2026 15:13:28 +0100 Subject: [PATCH 16/57] Fix heading format in md5_queries.md --- md5_queries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/md5_queries.md b/md5_queries.md index 63b1a60b..ccc7b6be 100644 --- a/md5_queries.md +++ b/md5_queries.md @@ -1,4 +1,4 @@ -giv## Queries for MD5 +## Queries for MD5 ### Initial From c86df8c235fee828c9532af89fda17dc9eec0b4e Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Wed, 4 Feb 2026 15:14:04 +0100 Subject: [PATCH 17/57] Enhance md5_queries.md with performance notes Added performance notes and clarified query patterns for MD5 filtering. --- md5_queries.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/md5_queries.md b/md5_queries.md index ccc7b6be..1b6abe67 100644 --- a/md5_queries.md +++ b/md5_queries.md @@ -74,14 +74,18 @@ List of hashes can get long (maximum (z*1000 items), giving more slow query 1] Spatial filtering with MD5 hash exclusion (GetGeometrySubset): WHERE ST_Centroid(ST_Envelope(geom_triangle)) && AND MD5(ST_AsBinary(geom_triangle)::text) NOT IN () + 2] MD5 hash filtering with spatial validation (FilterHashesByEnvelope): WHERE MD5(ST_AsBinary(geom_triangle)::text) IN () AND ST_Within(ST_Centroid(ST_Envelope(geom_triangle)), ) + 3] Hash-only filtering (GetGeometriesBoundingBox): WHERE MD5(ST_AsBinary(geom_triangle)::text) IN () Performance Notes: 1] The GIST spatial index handles the ST_Centroid(ST_Envelope(geom_triangle)) predicates + 2] The MD5 hash BTREE index handles the MD5(ST_AsBinary(geom_triangle)::text) IN/NOT IN predicates + 3] PostgreSQL will use both indexes (bitmap index scan) for queries with both predicates Optional: Materialized Hash Column From a76f0b1c91c3832f021aa4ff38fb97aaff944e8d Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Wed, 4 Feb 2026 15:21:05 +0100 Subject: [PATCH 18/57] Update md5_queries.md --- md5_queries.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/md5_queries.md b/md5_queries.md index 1b6abe67..125088a1 100644 --- a/md5_queries.md +++ b/md5_queries.md @@ -36,7 +36,19 @@ potential improvement: make exception for first tile on z=0 - do not filter on ## Tile 1_0_0.glb (level 1, x=0, y=0) -4] Count geometries in bounding box on level 1 excluding 1000 largest geometries from tile 0_0_0.glb (8 seconds!) +4] Filter the hashes from previous list to only geometries within tile 1_0_0 + +```sql + SELECT MD5(ST_AsBinary(geom)::text) as geom_hash + FROM bertt.nantes_reconstructed_buildings + WHERE MD5(ST_AsBinary(geom)::text) in ('9759cdee666f512a0c13df8245b667f9',... ) + AND ST_Within( + ST_Centroid(ST_Envelope(geom)), + ST_Transform(ST_MakeEnvelope(@xmin, @ymin, @xmax, @ymax, 4326), 5698) + ) +``` + +5] Count geometries in bounding box on level 1 excluding the geometries from tile 0_0_0.glb, including only the geometries within the tile ```sql SELECT COUNT(geom) FROM bertt.nantes_reconstructed_buildings WHERE ST_Centroid(ST_Envelope(geom)) && st_transform(ST_MakeEnvelope(-1.847105103048876, 47.14626198148698, -1.497208649149572, 47.384471872766284, 4326), 5698) AND MD5(ST_AsBinary(geom)::text) NOT IN ('9759cdee666f512a0c13df8245b667f9',..1000 items, ...) @@ -44,7 +56,7 @@ SELECT COUNT(geom) FROM bertt.nantes_reconstructed_buildings WHERE ST_Centroid(S Result: 235787 -5] Get geometries for tile 1_0_0.glb - 1000 largest geometries in tile 1_0_0 (10 seconds!) +6] Get geometries for tile 1_0_0.glb - 1000 largest geometries in tile 1_0_0 (10 seconds!) ```sql SELECT ST_AsBinary(st_transform(geom, 4978)), id , MD5(ST_AsBinary(geom)::text) as geom_hash FROM bertt.nantes_reconstructed_buildings where ST_Centroid(ST_Envelope(geom)) && st_transform(ST_MakeEnvelope(-1.847105103048876, 47.14626198148698, -1.497208649149572, 47.384471872766284, 4326), 5698) AND MD5(ST_AsBinary(geom)::text) NOT IN ('9759cdee666f512a0c13df8245b667f9', ..1000 items, ...) ORDER BY ST_Area(ST_Envelope(geom)) DESC LIMIT 1000 From 4d7c89295be84fee61519ebee94e5babb70e95ee Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Wed, 4 Feb 2026 15:26:38 +0100 Subject: [PATCH 19/57] fix octreetiler --- src/b3dm.tileset/GeometryRepository.cs | 6 +++++- src/b3dm.tileset/OctreeTiler.cs | 16 ++++++++++++---- src/b3dm.tileset/QuadtreeTiler.cs | 4 +++- src/pg2b3dm.database.tests/UnitTest1.cs | 2 +- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/b3dm.tileset/GeometryRepository.cs b/src/b3dm.tileset/GeometryRepository.cs index a898e0ff..e2233de4 100644 --- a/src/b3dm.tileset/GeometryRepository.cs +++ b/src/b3dm.tileset/GeometryRepository.cs @@ -25,13 +25,17 @@ public static HashSet FilterHashesByEnvelope(NpgsqlConnection conn, stri conn.Open(); + var envelope = keepProjection ? + $"ST_MakeEnvelope(@xmin, @ymin, @xmax, @ymax, {source_epsg})" : + $"ST_Transform(ST_MakeEnvelope(@xmin, @ymin, @xmax, @ymax, 4326), {source_epsg})"; + var query = $@" SELECT MD5(ST_AsBinary({geometryColumn})::text) as geom_hash FROM {tableName} WHERE MD5(ST_AsBinary({geometryColumn})::text) in ({hashList}) AND ST_Within( ST_Centroid(ST_Envelope({geometryColumn})), - ST_Transform(ST_MakeEnvelope(@xmin, @ymin, @xmax, @ymax, 4326), {source_epsg}) + {envelope} )"; using var cmd = new NpgsqlCommand(query, conn); diff --git a/src/b3dm.tileset/OctreeTiler.cs b/src/b3dm.tileset/OctreeTiler.cs index edcc6e9a..10800d90 100644 --- a/src/b3dm.tileset/OctreeTiler.cs +++ b/src/b3dm.tileset/OctreeTiler.cs @@ -57,7 +57,7 @@ public List GenerateTiles3D(BoundingBox3D bbox, int level, Tile3D tile, } else if (numberOfFeatures > tilingSettings.MaxFeaturesPerTile) { // First, create a tile with the largest geometries up to MaxFeaturesPerTile for this level - CreateTileForLargestGeometries3D(bbox, level, tile, tiles, tileBounds, where, processedGeometries); + var localProcessedGeometries = CreateTileForLargestGeometries3D(bbox, level, tile, tiles, tileBounds, where, processedGeometries); level++; for (var x = 0; x < 2; x++) { @@ -77,8 +77,11 @@ public List GenerateTiles3D(BoundingBox3D bbox, int level, Tile3D tile, var zend = z_start + dz; var bbox3d = new BoundingBox3D(xstart, ystart, z_start, xend, yend, zend); + var bboxOctant = new BoundingBox(xstart, ystart, xend, yend); + var filteredProcessedGeometries = GeometryRepository.FilterHashesByEnvelope(conn, inputTable.TableName, inputTable.GeometryColumn, bboxOctant, inputTable.EPSGCode, localProcessedGeometries, tilingSettings.KeepProjection); + var new_tile = new Tile3D(level, tile.X * 2 + x, tile.Y * 2 + y, tile.Z * 2 + z); - GenerateTiles3D(bbox3d, level, new_tile, tiles, tileBounds, processedGeometries); + GenerateTiles3D(bbox3d, level, new_tile, tiles, tileBounds, filteredProcessedGeometries); } } } @@ -91,8 +94,11 @@ public List GenerateTiles3D(BoundingBox3D bbox, int level, Tile3D tile, } - private void CreateTileForLargestGeometries3D(BoundingBox3D bbox, int level, Tile3D tile, List tiles, Dictionary tileBounds, string where, HashSet processedGeometries) + private HashSet CreateTileForLargestGeometries3D(BoundingBox3D bbox, int level, Tile3D tile, List tiles, Dictionary tileBounds, string where, HashSet processedGeometries) { + // clone processedIds to avoid modifying the original set in recursive calls + var localProcessedGeometries = new HashSet(processedGeometries); + // Get the largest geometries (up to MaxFeaturesPerTile) for this tile at this level int target_srs = tilingSettings.KeepProjection ? inputTable.EPSGCode : 4978; @@ -103,7 +109,7 @@ private void CreateTileForLargestGeometries3D(BoundingBox3D bbox, int level, Til // Collect hashes of processed geometries foreach (var geom in geometriesToProcess) { if (!string.IsNullOrEmpty(geom.Hash)) { - processedGeometries.Add(geom.Hash); + localProcessedGeometries.Add(geom.Hash); } } @@ -121,6 +127,8 @@ private void CreateTileForLargestGeometries3D(BoundingBox3D bbox, int level, Til var key = $"{tile.Level}_{tile.Z}_{tile.X}_{tile.Y}"; tileBounds[key] = bbox; } + + return localProcessedGeometries; } private void CreateTile3D(BoundingBox3D bbox, int level, Tile3D tile, List tiles, Dictionary tileBounds, string where, HashSet processedGeometries) diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index af56fd6e..99cb2208 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -184,7 +184,9 @@ private void CreateTile(BoundingBox bbox, Tile tile, List tiles, string wh } ProcessLodLevels(bbox, tile, lod, createGltf, keepProjection); - UpdateTileBoundingBox(tile, tileHashes, where, keepProjection); + if (!useImplicitTiling) { + UpdateTileBoundingBox(tile, tileHashes, where, keepProjection); + } tile.Available = true; diff --git a/src/pg2b3dm.database.tests/UnitTest1.cs b/src/pg2b3dm.database.tests/UnitTest1.cs index 789a74f7..1469d2d1 100644 --- a/src/pg2b3dm.database.tests/UnitTest1.cs +++ b/src/pg2b3dm.database.tests/UnitTest1.cs @@ -17,7 +17,7 @@ public class UnitTest1 public async Task Setup() { _containerPostgres = new PostgreSqlBuilder() - .WithImage("postgis/postgis:16-3.4-alpine") + .WithImage("postgis/postgis:18-3.6-alpine") .WithWaitStrategy(Wait.ForUnixContainer().UntilInternalTcpPortIsAvailable(5432)) .Build(); await _containerPostgres.StartAsync(); From a507f9c8d7579cecfe9af87d6923ba30a7c4af9e Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Wed, 4 Feb 2026 15:39:00 +0100 Subject: [PATCH 20/57] refactor writing tile --- src/b3dm.tileset/OctreeTiler.cs | 16 ++++------------ src/b3dm.tileset/QuadtreeTiler.cs | 12 ++++-------- src/b3dm.tileset/TileCreationHelper.cs | 22 ++++++++++++++++++++++ 3 files changed, 30 insertions(+), 20 deletions(-) create mode 100644 src/b3dm.tileset/TileCreationHelper.cs diff --git a/src/b3dm.tileset/OctreeTiler.cs b/src/b3dm.tileset/OctreeTiler.cs index 10800d90..d872e27d 100644 --- a/src/b3dm.tileset/OctreeTiler.cs +++ b/src/b3dm.tileset/OctreeTiler.cs @@ -113,12 +113,8 @@ private HashSet CreateTileForLargestGeometries3D(BoundingBox3D bbox, int } } - if (!tilingSettings.SkipCreateTiles) { - var bytes = TileWriter.ToTile(geometriesToProcess, tilesetSettings.Translation, copyright: tilesetSettings.Copyright, addOutlines: stylingSettings.AddOutlines, defaultColor: stylingSettings.DefaultColor, defaultMetallicRoughness: stylingSettings.DefaultMetallicRoughness, doubleSided: stylingSettings.DoubleSided, defaultAlphaMode: stylingSettings.DefaultAlphaMode, alphaCutoff: stylingSettings.AlphaCutoff, createGltf: tilingSettings.CreateGltf); - var file = $"{tilesetSettings.OutputSettings.ContentFolder}{Path.AltDirectorySeparatorChar}{tile.Level}_{tile.Z}_{tile.X}_{tile.Y}.glb"; - Console.Write($"\rCreating tile: {file} "); - File.WriteAllBytes($"{file}", bytes); - } + var file = $"{tilesetSettings.OutputSettings.ContentFolder}{Path.AltDirectorySeparatorChar}{tile.Level}_{tile.Z}_{tile.X}_{tile.Y}.glb"; + TileCreationHelper.WriteTileIfNeeded(geometriesToProcess, tilesetSettings.Translation, stylingSettings, tilesetSettings.Copyright, tilingSettings.CreateGltf, tilingSettings.SkipCreateTiles, file, file); tile.Available = true; } @@ -146,12 +142,8 @@ private void CreateTile3D(BoundingBox3D bbox, int level, Tile3D tile, List CreateTileForLargestGeometries(BoundingBox bbox, Tile ti tile.Lod = lod; - if (!skipCreateTiles) { - var bytes = TileWriter.ToTile(geometriesToProcess, translation, copyright: copyright, addOutlines: stylingSettings.AddOutlines, defaultColor: stylingSettings.DefaultColor, defaultMetallicRoughness: stylingSettings.DefaultMetallicRoughness, doubleSided: stylingSettings.DoubleSided, defaultAlphaMode: stylingSettings.DefaultAlphaMode, alphaCutoff: stylingSettings.AlphaCutoff, createGltf: createGltf); - File.WriteAllBytes($"{outputFolder}{Path.AltDirectorySeparatorChar}{file}", bytes); - } + var outputPath = $"{outputFolder}{Path.AltDirectorySeparatorChar}{file}"; + TileCreationHelper.WriteTileIfNeeded(geometriesToProcess, translation, stylingSettings, copyright, createGltf, skipCreateTiles, outputPath, file); ProcessLodLevels(bbox, tile, lod, createGltf, keepProjection); // todo: check the updateTileBoundingBox @@ -178,10 +176,8 @@ private void CreateTile(BoundingBox bbox, Tile tile, List tiles, string wh tile.Lod = lod; - if (!skipCreateTiles) { - var bytes = TileWriter.ToTile(geometries, translation, copyright: copyright, addOutlines: stylingSettings.AddOutlines, defaultColor: stylingSettings.DefaultColor, defaultMetallicRoughness: stylingSettings.DefaultMetallicRoughness, doubleSided: stylingSettings.DoubleSided, defaultAlphaMode: stylingSettings.DefaultAlphaMode, alphaCutoff: stylingSettings.AlphaCutoff, createGltf: createGltf); - File.WriteAllBytes($"{outputFolder}{Path.AltDirectorySeparatorChar}{file}", bytes); - } + var outputPath = $"{outputFolder}{Path.AltDirectorySeparatorChar}{file}"; + TileCreationHelper.WriteTileIfNeeded(geometries, translation, stylingSettings, copyright, createGltf, skipCreateTiles, outputPath, file); ProcessLodLevels(bbox, tile, lod, createGltf, keepProjection); if (!useImplicitTiling) { diff --git a/src/b3dm.tileset/TileCreationHelper.cs b/src/b3dm.tileset/TileCreationHelper.cs new file mode 100644 index 00000000..aa209a7a --- /dev/null +++ b/src/b3dm.tileset/TileCreationHelper.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.IO; +using B3dm.Tileset.settings; +using pg2b3dm; +using Wkb2Gltf; + +namespace B3dm.Tileset; + +public static class TileCreationHelper +{ + public static void WriteTileIfNeeded(List geometries, double[] translation, StylingSettings stylingSettings, string copyright, bool createGltf, bool skipCreateTiles, string outputPath, string displayName) + { + if (skipCreateTiles) { + return; + } + + var bytes = TileWriter.ToTile(geometries, translation, copyright: copyright, addOutlines: stylingSettings.AddOutlines, defaultColor: stylingSettings.DefaultColor, defaultMetallicRoughness: stylingSettings.DefaultMetallicRoughness, doubleSided: stylingSettings.DoubleSided, defaultAlphaMode: stylingSettings.DefaultAlphaMode, alphaCutoff: stylingSettings.AlphaCutoff, createGltf: createGltf); + Console.Write($"\rCreating tile: {displayName} "); + File.WriteAllBytes(outputPath, bytes); + } +} From 2dc043bc28b3d2afef0489c613ca89d116b5cb72 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Wed, 4 Feb 2026 15:50:37 +0100 Subject: [PATCH 21/57] Update md5_queries.md --- md5_queries.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/md5_queries.md b/md5_queries.md index 125088a1..4796e517 100644 --- a/md5_queries.md +++ b/md5_queries.md @@ -104,11 +104,6 @@ List of hashes can get long (maximum (z*1000 items), giving more slow query ## Todo -- Check spatial indexes - -- limit 'not in' list of md5 hashes from above level, e.g. only use the hashes of the geometries that intersect the tile envelope. -Will probably help a bit but not enough (list of hashes stays long) - - idea: make a temporary blacklist table with the to be exluded hashes? - idea: force use of id column (longs)? From c22f1845d0a7a7772ed7031509168f6b266b9e94 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 08:16:46 +0000 Subject: [PATCH 22/57] Initial plan From 9a5a388e04e8fad2542c6b657db96b5aae71e16b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 08:21:42 +0000 Subject: [PATCH 23/57] Replace string concatenation with parameterized queries using ANY operator Co-authored-by: bertt <538812+bertt@users.noreply.github.com> --- src/b3dm.tileset/FeatureCountRepository.cs | 11 +++++- src/b3dm.tileset/GeometryRepository.cs | 39 +++++++++++----------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/b3dm.tileset/FeatureCountRepository.cs b/src/b3dm.tileset/FeatureCountRepository.cs index 5e582dd3..6cbd1f6e 100644 --- a/src/b3dm.tileset/FeatureCountRepository.cs +++ b/src/b3dm.tileset/FeatureCountRepository.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; using Npgsql; using Wkx; @@ -9,11 +10,19 @@ public static class FeatureCountRepository public static int CountFeaturesInBox(NpgsqlConnection conn, string geometry_table, string geometry_column, Point from, Point to, string query, int source_epsg, bool keepProjection = false, HashSet excludeHashes = null) { var select = $"COUNT({geometry_column})"; - var where = GeometryRepository.GetWhere(geometry_column, from, to, query, source_epsg, keepProjection, excludeHashes); + var where = GeometryRepository.GetWhere(geometry_column, from, to, query, source_epsg, keepProjection); + + // Add hash exclusion filter using parameterized query + if (excludeHashes != null && excludeHashes.Count > 0) { + where += $" AND MD5(ST_AsBinary({geometry_column})::text) != ALL(@excludeHashes)"; + } var sql = $"SELECT {select} FROM {geometry_table} WHERE {where}"; conn.Open(); var cmd = new NpgsqlCommand(sql, conn); + if (excludeHashes != null && excludeHashes.Count > 0) { + cmd.Parameters.AddWithValue("excludeHashes", excludeHashes.ToArray()); + } var reader = cmd.ExecuteReader(); reader.Read(); var count = reader.GetInt32(0); diff --git a/src/b3dm.tileset/GeometryRepository.cs b/src/b3dm.tileset/GeometryRepository.cs index e2233de4..a9b6614b 100644 --- a/src/b3dm.tileset/GeometryRepository.cs +++ b/src/b3dm.tileset/GeometryRepository.cs @@ -21,8 +21,6 @@ public static HashSet FilterHashesByEnvelope(NpgsqlConnection conn, stri var filteredHashes = new HashSet(); - var hashList = string.Join(",", geometryHashes.Select(h => $"'{h}'")); - conn.Open(); var envelope = keepProjection ? @@ -32,7 +30,7 @@ public static HashSet FilterHashesByEnvelope(NpgsqlConnection conn, stri var query = $@" SELECT MD5(ST_AsBinary({geometryColumn})::text) as geom_hash FROM {tableName} - WHERE MD5(ST_AsBinary({geometryColumn})::text) in ({hashList}) + WHERE MD5(ST_AsBinary({geometryColumn})::text) = ANY(@hashes) AND ST_Within( ST_Centroid(ST_Envelope({geometryColumn})), {envelope} @@ -65,13 +63,12 @@ public static double[] GetGeometriesBoundingBox(NpgsqlConnection conn, string ge $"select st_Asbinary(st_3dextent({geometry_column})) ": $"select st_Asbinary(st_3dextent(st_transform({geometry_column}, 4979))) "; - var hashList = string.Join(",", tileHashes.Select(h => $"'{h}'")); - - var sqlWhere = $" MD5(ST_AsBinary({geometry_column})::text) in ({hashList})"; + var sqlWhere = $" MD5(ST_AsBinary({geometry_column})::text) = ANY(@hashes)"; var sql = $"{sqlSelect} from {geometry_table} where {sqlWhere}"; conn.Open(); var cmd = new NpgsqlCommand(sql, conn); + cmd.Parameters.AddWithValue("hashes", tileHashes.ToArray()); var reader = cmd.ExecuteReader(); reader.Read(); var stream = reader.GetStream(0); @@ -92,16 +89,29 @@ public static List GetGeometrySubset(NpgsqlConnection conn, stri // todo: fix unit test when there is no z var points = GetPoints(bbox); - var sqlWhere = GetWhere(geometry_column, points.fromPoint, points.toPoint, query, source_epsg, keepProjection, excludeHashes); + var sqlWhere = GetWhere(geometry_column, points.fromPoint, points.toPoint, query, source_epsg, keepProjection); + + // Add hash exclusion filter using parameterized query + if (excludeHashes != null && excludeHashes.Count > 0) { + sqlWhere += $" AND MD5(ST_AsBinary({geometry_column})::text) != ALL(@excludeHashes)"; + } + var sqlOrderBy = GetOrderBy(geometry_column); var sqlLimit = maxFeatures.HasValue ? $" LIMIT {maxFeatures.Value}" : ""; var sql = sqlselect + sqlFrom + " where " + sqlWhere + sqlOrderBy + sqlLimit; - var geometries = GetGeometries(conn, shaderColumn, attributesColumns, sql, radiusColumn, geometry_column); + conn.Open(); + var cmd = new NpgsqlCommand(sql, conn); + if (excludeHashes != null && excludeHashes.Count > 0) { + cmd.Parameters.AddWithValue("excludeHashes", excludeHashes.ToArray()); + } + + var geometries = GetGeometries(cmd, shaderColumn, attributesColumns, radiusColumn, geometry_column); + conn.Close(); return geometries; } - public static string GetWhere(string geometry_column, Point from, Point to, string query, int source_epsg, bool keepProjection, HashSet excludeHashes = null) + public static string GetWhere(string geometry_column, Point from, Point to, string query, int source_epsg, bool keepProjection) { var fromX = from.X.Value.ToString(CultureInfo.InvariantCulture); var fromY = from.Y.Value.ToString(CultureInfo.InvariantCulture); @@ -130,12 +140,6 @@ public static string GetWhere(string geometry_column, Point from, Point to, stri $"ST_3DIntersects({geom}, st_transform(ST_3DMakeBox({fromBox}, {toBox}), {source_epsg})) {query}"; } - // Add hash exclusion filter - if (excludeHashes != null && excludeHashes.Count > 0) { - var hashList = string.Join(",", excludeHashes.Select(h => $"'{h}'")); - where += $" AND MD5(ST_AsBinary({geometry_column})::text) NOT IN ({hashList})"; - } - return where; } @@ -168,11 +172,9 @@ public static string GetOrderBy(string geometry_column) return $" ORDER BY ST_Area(ST_Envelope({geometry_column})) DESC"; } - public static List GetGeometries(NpgsqlConnection conn, string shaderColumn, string attributesColumns, string sql, string radiusColumn, string geometry_column = "") + public static List GetGeometries(NpgsqlCommand cmd, string shaderColumn, string attributesColumns, string radiusColumn, string geometry_column = "") { var geometries = new List(); - conn.Open(); - var cmd = new NpgsqlCommand(sql, conn); var reader = cmd.ExecuteReader(); var attributesColumnIds = new Dictionary(); var shadersColumnId = int.MinValue; @@ -230,7 +232,6 @@ public static List GetGeometries(NpgsqlConnection conn, string s } reader.Close(); - conn.Close(); return geometries; } From 96c10afeb521429ba571a075653e293166941c62 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 08:23:08 +0000 Subject: [PATCH 24/57] Update md5_queries.md to reflect parameterized query solution Co-authored-by: bertt <538812+bertt@users.noreply.github.com> --- md5_queries.md | 47 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/md5_queries.md b/md5_queries.md index 4796e517..826f860d 100644 --- a/md5_queries.md +++ b/md5_queries.md @@ -41,30 +41,38 @@ potential improvement: make exception for first tile on z=0 - do not filter on ```sql SELECT MD5(ST_AsBinary(geom)::text) as geom_hash FROM bertt.nantes_reconstructed_buildings - WHERE MD5(ST_AsBinary(geom)::text) in ('9759cdee666f512a0c13df8245b667f9',... ) + WHERE MD5(ST_AsBinary(geom)::text) = ANY($1) AND ST_Within( ST_Centroid(ST_Envelope(geom)), - ST_Transform(ST_MakeEnvelope(@xmin, @ymin, @xmax, @ymax, 4326), 5698) + ST_Transform(ST_MakeEnvelope($2, $3, $4, $5, 4326), 5698) ) ``` +Note: Using parameterized query with array parameter instead of string concatenation. + 5] Count geometries in bounding box on level 1 excluding the geometries from tile 0_0_0.glb, including only the geometries within the tile ```sql -SELECT COUNT(geom) FROM bertt.nantes_reconstructed_buildings WHERE ST_Centroid(ST_Envelope(geom)) && st_transform(ST_MakeEnvelope(-1.847105103048876, 47.14626198148698, -1.497208649149572, 47.384471872766284, 4326), 5698) AND MD5(ST_AsBinary(geom)::text) NOT IN ('9759cdee666f512a0c13df8245b667f9',..1000 items, ...) +SELECT COUNT(geom) FROM bertt.nantes_reconstructed_buildings WHERE ST_Centroid(ST_Envelope(geom)) && st_transform(ST_MakeEnvelope(-1.847105103048876, 47.14626198148698, -1.497208649149572, 47.384471872766284, 4326), 5698) AND MD5(ST_AsBinary(geom)::text) != ALL($1) ``` +Note: Using parameterized query with array parameter instead of string concatenation. + Result: 235787 -6] Get geometries for tile 1_0_0.glb - 1000 largest geometries in tile 1_0_0 (10 seconds!) +6] Get geometries for tile 1_0_0.glb - 1000 largest geometries in tile 1_0_0 ```sql -SELECT ST_AsBinary(st_transform(geom, 4978)), id , MD5(ST_AsBinary(geom)::text) as geom_hash FROM bertt.nantes_reconstructed_buildings where ST_Centroid(ST_Envelope(geom)) && st_transform(ST_MakeEnvelope(-1.847105103048876, 47.14626198148698, -1.497208649149572, 47.384471872766284, 4326), 5698) AND MD5(ST_AsBinary(geom)::text) NOT IN ('9759cdee666f512a0c13df8245b667f9', ..1000 items, ...) ORDER BY ST_Area(ST_Envelope(geom)) DESC LIMIT 1000 +SELECT ST_AsBinary(st_transform(geom, 4978)), id , MD5(ST_AsBinary(geom)::text) as geom_hash FROM bertt.nantes_reconstructed_buildings where ST_Centroid(ST_Envelope(geom)) && st_transform(ST_MakeEnvelope(-1.847105103048876, 47.14626198148698, -1.497208649149572, 47.384471872766284, 4326), 5698) AND MD5(ST_AsBinary(geom)::text) != ALL($1) ORDER BY ST_Area(ST_Envelope(geom)) DESC LIMIT 1000 ``` +Note: Using parameterized query with array parameter instead of string concatenation. + ## Issue -List of hashes can get long (maximum (z*1000 items), giving more slow query +List of hashes can get long (maximum z*1000 items). Previously this was handled with string concatenation which could lead to performance issues and potential SQL injection vulnerabilities. + +**Solution**: Now using parameterized queries with PostgreSQL's `= ANY()` and `!= ALL()` operators for better performance and security. ## Spatial indexing @@ -85,27 +93,42 @@ List of hashes can get long (maximum (z*1000 items), giving more slow query The queries now use three main patterns: 1] Spatial filtering with MD5 hash exclusion (GetGeometrySubset): WHERE ST_Centroid(ST_Envelope(geom_triangle)) && - AND MD5(ST_AsBinary(geom_triangle)::text) NOT IN () + AND MD5(ST_AsBinary(geom_triangle)::text) != ALL($1) - 2] MD5 hash filtering with spatial validation (FilterHashesByEnvelope): WHERE MD5(ST_AsBinary(geom_triangle)::text) IN () + 2] MD5 hash filtering with spatial validation (FilterHashesByEnvelope): WHERE MD5(ST_AsBinary(geom_triangle)::text) = ANY($1) AND ST_Within(ST_Centroid(ST_Envelope(geom_triangle)), ) - 3] Hash-only filtering (GetGeometriesBoundingBox): WHERE MD5(ST_AsBinary(geom_triangle)::text) IN () + 3] Hash-only filtering (GetGeometriesBoundingBox): WHERE MD5(ST_AsBinary(geom_triangle)::text) = ANY($1) Performance Notes: 1] The GIST spatial index handles the ST_Centroid(ST_Envelope(geom_triangle)) predicates - 2] The MD5 hash BTREE index handles the MD5(ST_AsBinary(geom_triangle)::text) IN/NOT IN predicates + 2] The MD5 hash BTREE index handles the MD5(ST_AsBinary(geom_triangle)::text) = ANY/!= ALL predicates 3] PostgreSQL will use both indexes (bitmap index scan) for queries with both predicates + + 4] Using parameterized queries with ANY/ALL operators provides better performance than string-concatenated IN/NOT IN clauses Optional: Materialized Hash Column +## Solution + +The hash filtering now uses PostgreSQL's `= ANY(@param)` operator with array parameters instead of string concatenation: + +1. **Hash Inclusion (IN clause)**: Changed from `MD5(...) IN ('hash1', 'hash2', ...)` to `MD5(...) = ANY(@hashes)` with parameterized array +2. **Hash Exclusion (NOT IN clause)**: Changed from `MD5(...) NOT IN ('hash1', 'hash2', ...)` to `MD5(...) != ALL(@excludeHashes)` with parameterized array + +Benefits: +- Eliminates SQL injection risk (even though MD5 hashes are predictable) +- Better performance with large hash lists +- Cleaner, more maintainable code +- Proper use of parameterized queries + ## Todo -- idea: make a temporary blacklist table with the to be exluded hashes? +- ~~idea: make a temporary blacklist table with the to be exluded hashes?~~ (Solved using parameterized arrays) - idea: force use of id column (longs)? -- Other solutions? +- ~~Other solutions?~~ (Implemented using `ANY` and `ALL` operators) From b2cdb4da4c20e424db4342cca98096c019de3b27 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 08:25:01 +0000 Subject: [PATCH 25/57] Improve connection management with try-finally blocks Co-authored-by: bertt <538812+bertt@users.noreply.github.com> --- src/b3dm.tileset/FeatureCountRepository.cs | 21 +++-- src/b3dm.tileset/GeometryRepository.cs | 100 +++++++++++---------- 2 files changed, 66 insertions(+), 55 deletions(-) diff --git a/src/b3dm.tileset/FeatureCountRepository.cs b/src/b3dm.tileset/FeatureCountRepository.cs index 6cbd1f6e..c21af284 100644 --- a/src/b3dm.tileset/FeatureCountRepository.cs +++ b/src/b3dm.tileset/FeatureCountRepository.cs @@ -19,15 +19,18 @@ public static int CountFeaturesInBox(NpgsqlConnection conn, string geometry_tabl var sql = $"SELECT {select} FROM {geometry_table} WHERE {where}"; conn.Open(); - var cmd = new NpgsqlCommand(sql, conn); - if (excludeHashes != null && excludeHashes.Count > 0) { - cmd.Parameters.AddWithValue("excludeHashes", excludeHashes.ToArray()); + try { + using var cmd = new NpgsqlCommand(sql, conn); + if (excludeHashes != null && excludeHashes.Count > 0) { + cmd.Parameters.AddWithValue("excludeHashes", excludeHashes.ToArray()); + } + using var reader = cmd.ExecuteReader(); + reader.Read(); + var count = reader.GetInt32(0); + return count; + } + finally { + conn.Close(); } - var reader = cmd.ExecuteReader(); - reader.Read(); - var count = reader.GetInt32(0); - reader.Close(); - conn.Close(); - return count; } } diff --git a/src/b3dm.tileset/GeometryRepository.cs b/src/b3dm.tileset/GeometryRepository.cs index a9b6614b..8b9443d6 100644 --- a/src/b3dm.tileset/GeometryRepository.cs +++ b/src/b3dm.tileset/GeometryRepository.cs @@ -22,36 +22,38 @@ public static HashSet FilterHashesByEnvelope(NpgsqlConnection conn, stri var filteredHashes = new HashSet(); conn.Open(); + try { + var envelope = keepProjection ? + $"ST_MakeEnvelope(@xmin, @ymin, @xmax, @ymax, {source_epsg})" : + $"ST_Transform(ST_MakeEnvelope(@xmin, @ymin, @xmax, @ymax, 4326), {source_epsg})"; + + var query = $@" + SELECT MD5(ST_AsBinary({geometryColumn})::text) as geom_hash + FROM {tableName} + WHERE MD5(ST_AsBinary({geometryColumn})::text) = ANY(@hashes) + AND ST_Within( + ST_Centroid(ST_Envelope({geometryColumn})), + {envelope} + )"; + + using var cmd = new NpgsqlCommand(query, conn); + cmd.Parameters.AddWithValue("hashes", geometryHashes.ToArray()); + cmd.Parameters.AddWithValue("xmin", bbox.XMin); + cmd.Parameters.AddWithValue("ymin", bbox.YMin); + cmd.Parameters.AddWithValue("xmax", bbox.XMax); + cmd.Parameters.AddWithValue("ymax", bbox.YMax); + + using var reader = cmd.ExecuteReader(); + while (reader.Read()) { + var hash = reader.GetString(0); + filteredHashes.Add(hash); + } - var envelope = keepProjection ? - $"ST_MakeEnvelope(@xmin, @ymin, @xmax, @ymax, {source_epsg})" : - $"ST_Transform(ST_MakeEnvelope(@xmin, @ymin, @xmax, @ymax, 4326), {source_epsg})"; - - var query = $@" - SELECT MD5(ST_AsBinary({geometryColumn})::text) as geom_hash - FROM {tableName} - WHERE MD5(ST_AsBinary({geometryColumn})::text) = ANY(@hashes) - AND ST_Within( - ST_Centroid(ST_Envelope({geometryColumn})), - {envelope} - )"; - - using var cmd = new NpgsqlCommand(query, conn); - cmd.Parameters.AddWithValue("hashes", geometryHashes.ToArray()); - cmd.Parameters.AddWithValue("xmin", bbox.XMin); - cmd.Parameters.AddWithValue("ymin", bbox.YMin); - cmd.Parameters.AddWithValue("xmax", bbox.XMax); - cmd.Parameters.AddWithValue("ymax", bbox.YMax); - - using var reader = cmd.ExecuteReader(); - while (reader.Read()) { - var hash = reader.GetString(0); - filteredHashes.Add(hash); + return filteredHashes; + } + finally { + conn.Close(); } - - conn.Close(); - - return filteredHashes; } /// @@ -67,18 +69,20 @@ public static double[] GetGeometriesBoundingBox(NpgsqlConnection conn, string ge var sql = $"{sqlSelect} from {geometry_table} where {sqlWhere}"; conn.Open(); - var cmd = new NpgsqlCommand(sql, conn); - cmd.Parameters.AddWithValue("hashes", tileHashes.ToArray()); - var reader = cmd.ExecuteReader(); - reader.Read(); - var stream = reader.GetStream(0); - var geometry = Geometry.Deserialize(stream); - var result = BBox3D.GetBoundingBoxPoints(geometry); - - reader.Close(); - conn.Close(); + try { + using var cmd = new NpgsqlCommand(sql, conn); + cmd.Parameters.AddWithValue("hashes", tileHashes.ToArray()); + using var reader = cmd.ExecuteReader(); + reader.Read(); + var stream = reader.GetStream(0); + var geometry = Geometry.Deserialize(stream); + var result = BBox3D.GetBoundingBoxPoints(geometry); - return result; + return result; + } + finally { + conn.Close(); + } } public static List GetGeometrySubset(NpgsqlConnection conn, string geometry_table, string geometry_column, double[] bbox, int source_epsg, int target_srs, string shaderColumn = "", string attributesColumns = "", string query = "", string radiusColumn = "", bool keepProjection = false, HashSet excludeHashes = null, int? maxFeatures = null) @@ -101,14 +105,18 @@ public static List GetGeometrySubset(NpgsqlConnection conn, stri var sql = sqlselect + sqlFrom + " where " + sqlWhere + sqlOrderBy + sqlLimit; conn.Open(); - var cmd = new NpgsqlCommand(sql, conn); - if (excludeHashes != null && excludeHashes.Count > 0) { - cmd.Parameters.AddWithValue("excludeHashes", excludeHashes.ToArray()); + try { + using var cmd = new NpgsqlCommand(sql, conn); + if (excludeHashes != null && excludeHashes.Count > 0) { + cmd.Parameters.AddWithValue("excludeHashes", excludeHashes.ToArray()); + } + + var geometries = GetGeometries(cmd, shaderColumn, attributesColumns, radiusColumn, geometry_column); + return geometries; + } + finally { + conn.Close(); } - - var geometries = GetGeometries(cmd, shaderColumn, attributesColumns, radiusColumn, geometry_column); - conn.Close(); - return geometries; } public static string GetWhere(string geometry_column, Point from, Point to, string query, int source_epsg, bool keepProjection) From 4182510ac2b56f3ad1cfdff695b451a29c7aded8 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 09:27:51 +0100 Subject: [PATCH 26/57] Update src/b3dm.tileset/SpatialIndexChecker.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/b3dm.tileset/SpatialIndexChecker.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/b3dm.tileset/SpatialIndexChecker.cs b/src/b3dm.tileset/SpatialIndexChecker.cs index 496866a1..6d8fd607 100644 --- a/src/b3dm.tileset/SpatialIndexChecker.cs +++ b/src/b3dm.tileset/SpatialIndexChecker.cs @@ -20,7 +20,7 @@ public static bool HasSpatialIndex(NpgsqlConnection conn, string geometry_table, cmd.Parameters.AddWithValue("schema", schema); cmd.Parameters.AddWithValue("geometry_table", geometry_table); - cmd.Parameters.AddWithValue("index", "%st_centroid(st_envelope(" + geometry_column + "%)), st_envelope(" + geometry_column + "%))%"); + cmd.Parameters.AddWithValue("index", "%st_centroid(st_envelope(" + geometry_column + "%))%"); var reader = cmd.ExecuteReader(); reader.Read(); From 4be180c4cc457673b6a28fdc6471ccc613535eef Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 09:36:34 +0100 Subject: [PATCH 27/57] Update README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f0f2df2b..72d2b0de 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ triangulation. Geometries with interior rings are supported. For large datasets create a spatial index on the geometry column: ``` -psql> CREATE INDEX ON the_table USING gist(st_centroid(st_envelope(geom)), st_envelope(geom)); +psql> CREATE INDEX ON the_table USING gist (st_envelope(geom)); ``` When there the spatial index is not present the following warning is shown. From 80085d2e4d2b22cb11088f5e8a8c6acb97d33c09 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 10:25:09 +0100 Subject: [PATCH 28/57] add md5 to index check --- src/b3dm.tileset/SpatialIndexChecker.cs | 17 +++++++++++++++-- src/pg2b3dm/Program.cs | 17 ++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/b3dm.tileset/SpatialIndexChecker.cs b/src/b3dm.tileset/SpatialIndexChecker.cs index 6d8fd607..21bd650e 100644 --- a/src/b3dm.tileset/SpatialIndexChecker.cs +++ b/src/b3dm.tileset/SpatialIndexChecker.cs @@ -4,9 +4,22 @@ namespace B3dm.Tileset; public static class SpatialIndexChecker { public static bool HasSpatialIndex(NpgsqlConnection conn, string geometry_table, string geometry_column) + { + var indexDef = $"%st_centroid(st_envelope({geometry_column}))%"; + + return HasIndex(conn, geometry_table, indexDef); + } + + public static bool HashMd5Index(NpgsqlConnection conn, string geometry_table, string geometry_column) + { + var indexDef = $"%md5((st_asbinary({geometry_column}))::text%)"; + return HasIndex(conn, geometry_table, indexDef); + } + + private static bool HasIndex(NpgsqlConnection conn, string geometry_table, string indexDef) { var schema = "public"; - if(geometry_table.Contains('.')) { + if (geometry_table.Contains('.')) { var items = geometry_table.Split('.', 2); schema = items[0]; geometry_table = items[1]; @@ -20,7 +33,7 @@ public static bool HasSpatialIndex(NpgsqlConnection conn, string geometry_table, cmd.Parameters.AddWithValue("schema", schema); cmd.Parameters.AddWithValue("geometry_table", geometry_table); - cmd.Parameters.AddWithValue("index", "%st_centroid(st_envelope(" + geometry_column + "%))%"); + cmd.Parameters.AddWithValue("index", indexDef); var reader = cmd.ExecuteReader(); reader.Read(); diff --git a/src/pg2b3dm/Program.cs b/src/pg2b3dm/Program.cs index cd5f7d6e..45591c42 100644 --- a/src/pg2b3dm/Program.cs +++ b/src/pg2b3dm/Program.cs @@ -96,7 +96,7 @@ static void Main(string[] args) Console.WriteLine("-----------------------------------------------------------------------------"); Console.WriteLine($"WARNING: No spatial index detected on {inputTable.TableName}.{inputTable.GeometryColumn}"); Console.WriteLine("Fix: add a spatial index, for example: "); - Console.WriteLine($"'CREATE INDEX ON {inputTable.TableName} USING gist(st_centroid(st_envelope({inputTable.GeometryColumn})), st_envelope({inputTable.GeometryColumn}))'"); + Console.WriteLine($"'CREATE INDEX ON {inputTable.TableName} USING gist(st_centroid(st_envelope({inputTable.GeometryColumn})))'"); Console.WriteLine("-----------------------------------------------------------------------------"); Console.WriteLine(); } @@ -104,6 +104,21 @@ static void Main(string[] args) Console.WriteLine($"Spatial index detected on {inputTable.TableName}.{inputTable.GeometryColumn}"); } + // Check md5 index + var hasMd5Index = SpatialIndexChecker.HashMd5Index(conn, inputTable.TableName, inputTable.GeometryColumn); + if (!hasMd5Index) { + Console.WriteLine(); + Console.WriteLine("-----------------------------------------------------------------------------"); + Console.WriteLine($"WARNING: No md5 index detected on {inputTable.TableName}.{inputTable.GeometryColumn}"); + Console.WriteLine("Fix: add a md5 index, for example: "); + Console.WriteLine($"'CREATE INDEX ON {inputTable.TableName} using btree(md5(st_asbinary({inputTable.GeometryColumn})::text))'"); + Console.WriteLine("-----------------------------------------------------------------------------"); + Console.WriteLine(); + } + else { + Console.WriteLine($"Md5 index detected on {inputTable.TableName}.{inputTable.GeometryColumn}"); + } + var skipCreateTiles = (bool)o.SkipCreateTiles; Console.WriteLine("Skip create tiles: " + skipCreateTiles); From d4a1cd7e0e124cd82454eedcdfa87c3d2be4ff9d Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 10:32:39 +0100 Subject: [PATCH 29/57] fix releative path --- src/pg2b3dm.sln | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg2b3dm.sln b/src/pg2b3dm.sln index 31bec38b..6f234a34 100644 --- a/src/pg2b3dm.sln +++ b/src/pg2b3dm.sln @@ -24,7 +24,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "doc", "doc", "{0403ED2E-B5A ..\..\..\..\..\Users\bertt\Desktop\delaware.png = ..\..\..\..\..\Users\bertt\Desktop\delaware.png ..\..\..\..\..\Users\bert\Desktop\demo_pg2b3dm.gif = ..\..\..\..\..\Users\bert\Desktop\demo_pg2b3dm.gif ..\getting_started.md = ..\getting_started.md - C:\Users\bert\Desktop\test.csv\materials_for_features.csv = C:\Users\bert\Desktop\test.csv\materials_for_features.csv + ..\dataprocessing\materials_for_features.csv = ..\dataprocessing\materials_for_features.csv md5_queries.md = md5_queries.md ..\README.md = ..\README.md release_notes_0.10.md = release_notes_0.10.md From b94d6e53e3cc878d7ce91d20b134192afec605b1 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 10:36:18 +0100 Subject: [PATCH 30/57] skip redundant check --- src/b3dm.tileset/QuadtreeTiler.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index 68453d13..b8ddf342 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -185,10 +185,6 @@ private void CreateTile(BoundingBox bbox, Tile tile, List tiles, string wh } tile.Available = true; - - if (skipCreateTiles) { - tile.Available = true; - } } else { tile.Available = false; From 3dbcfbd8050e646482115382d7556b13b3d3e692 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 10:45:06 +0100 Subject: [PATCH 31/57] Update src/b3dm.tileset/QuadtreeTiler.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/b3dm.tileset/QuadtreeTiler.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index b8ddf342..b93a93bd 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -167,11 +167,9 @@ private void CreateTile(BoundingBox bbox, Tile tile, List tiles, string wh if (geometries.Count > 0) { // Collect hashes of processed geometries - foreach (var geom in geometries) { - if (!string.IsNullOrEmpty(geom.Hash)) { - tileHashes.Add(geom.Hash); - processedGeometries.Add(geom.Hash); - } + foreach (var geom in geometries.Where(g => !string.IsNullOrEmpty(g.Hash))) { + tileHashes.Add(geom.Hash); + processedGeometries.Add(geom.Hash); } tile.Lod = lod; From 39d262ff1d90e3d7c22a458a162872bc69b4c5f9 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 10:46:08 +0100 Subject: [PATCH 32/57] Update src/b3dm.tileset/OctreeTiler.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/b3dm.tileset/OctreeTiler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/b3dm.tileset/OctreeTiler.cs b/src/b3dm.tileset/OctreeTiler.cs index d872e27d..be563527 100644 --- a/src/b3dm.tileset/OctreeTiler.cs +++ b/src/b3dm.tileset/OctreeTiler.cs @@ -96,7 +96,7 @@ public List GenerateTiles3D(BoundingBox3D bbox, int level, Tile3D tile, private HashSet CreateTileForLargestGeometries3D(BoundingBox3D bbox, int level, Tile3D tile, List tiles, Dictionary tileBounds, string where, HashSet processedGeometries) { - // clone processedIds to avoid modifying the original set in recursive calls + // clone processedGeometries to avoid modifying the original set in recursive calls var localProcessedGeometries = new HashSet(processedGeometries); // Get the largest geometries (up to MaxFeaturesPerTile) for this tile at this level From c73accf481cf473f212d38cbd05e18fc4f22bbe0 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 10:46:48 +0100 Subject: [PATCH 33/57] Update src/b3dm.tileset/QuadtreeTiler.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/b3dm.tileset/QuadtreeTiler.cs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index b93a93bd..5c9c8c87 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -192,18 +192,16 @@ private void CreateTile(BoundingBox bbox, Tile tile, List tiles, string wh private void ProcessLodLevels(BoundingBox bbox, Tile tile, int lod, bool createGltf, bool keepProjection) { - if (inputTable.LodColumn != String.Empty) { - if (lod < lods.Max()) { - // take the next lod - var currentIndex = lods.FindIndex(p => p == lod); - var nextIndex = currentIndex + 1; - var nextLod = lods[nextIndex]; - // make a copy of the tile - var t2 = new Tile(tile.Z, tile.X, tile.Y); - t2.BoundingBox = tile.BoundingBox; - var lodNextTiles = GenerateTiles(bbox, t2, new List(), nextLod, createGltf, keepProjection); - tile.Children = lodNextTiles; - } + if (inputTable.LodColumn != String.Empty && lod < lods.Max()) { + // take the next lod + var currentIndex = lods.FindIndex(p => p == lod); + var nextIndex = currentIndex + 1; + var nextLod = lods[nextIndex]; + // make a copy of the tile + var t2 = new Tile(tile.Z, tile.X, tile.Y); + t2.BoundingBox = tile.BoundingBox; + var lodNextTiles = GenerateTiles(bbox, t2, new List(), nextLod, createGltf, keepProjection); + tile.Children = lodNextTiles; } } From 88d5f9e33e7af9468aa9b6d453cd396b076d7b12 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 10:47:07 +0100 Subject: [PATCH 34/57] Update src/b3dm.tileset/QuadtreeTiler.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/b3dm.tileset/QuadtreeTiler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index 5c9c8c87..d495cbe6 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -23,7 +23,7 @@ public class QuadtreeTiler private readonly bool skipCreateTiles; private readonly StylingSettings stylingSettings; private InputTable inputTable; - private bool useImplicitTiling = false; + private readonly bool useImplicitTiling; public QuadtreeTiler(string connectionString, InputTable inputTable, StylingSettings stylingSettings, int maxFeaturesPerTile, double[] translation, string outputFolder, List lods, string copyright = "", bool skipCreateTiles = false, bool useImplicitTiling=false) { From b67f67bc092a646aa00e48876d56c56f6e461569 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 10:48:47 +0100 Subject: [PATCH 35/57] Update src/b3dm.tileset/QuadtreeTiler.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/b3dm.tileset/QuadtreeTiler.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index d495cbe6..b04fe6a3 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -111,11 +111,9 @@ private HashSet CreateTileForLargestGeometries(BoundingBox bbox, Tile ti if (geometriesToProcess.Count > 0) { // Collect hashes of processed geometries - foreach (var geom in geometriesToProcess) { - if (!string.IsNullOrEmpty(geom.Hash)) { - localProcessedGeometries.Add(geom.Hash); - tileHashes.Add(geom.Hash); - } + foreach (var geom in geometriesToProcess.Where(geom => !string.IsNullOrEmpty(geom.Hash))) { + localProcessedGeometries.Add(geom.Hash); + tileHashes.Add(geom.Hash); } var file = $"{tile.Z}_{tile.X}_{tile.Y}"; From 47d38ed304f45715002f274b131675c8082a8e20 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 10:50:45 +0100 Subject: [PATCH 36/57] update octreetiler --- src/b3dm.tileset/OctreeTiler.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/b3dm.tileset/OctreeTiler.cs b/src/b3dm.tileset/OctreeTiler.cs index be563527..11411c03 100644 --- a/src/b3dm.tileset/OctreeTiler.cs +++ b/src/b3dm.tileset/OctreeTiler.cs @@ -98,6 +98,7 @@ private HashSet CreateTileForLargestGeometries3D(BoundingBox3D bbox, int { // clone processedGeometries to avoid modifying the original set in recursive calls var localProcessedGeometries = new HashSet(processedGeometries); + var tileHashes = new HashSet(); // Get the largest geometries (up to MaxFeaturesPerTile) for this tile at this level int target_srs = tilingSettings.KeepProjection ? inputTable.EPSGCode : 4978; @@ -106,11 +107,9 @@ private HashSet CreateTileForLargestGeometries3D(BoundingBox3D bbox, int var geometriesToProcess = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, bbox1, inputTable.EPSGCode, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, tilingSettings.KeepProjection, processedGeometries, tilingSettings.MaxFeaturesPerTile); if (geometriesToProcess.Count > 0) { - // Collect hashes of processed geometries - foreach (var geom in geometriesToProcess) { - if (!string.IsNullOrEmpty(geom.Hash)) { - localProcessedGeometries.Add(geom.Hash); - } + foreach (var geom in geometriesToProcess.Where(geom => !string.IsNullOrEmpty(geom.Hash))) { + localProcessedGeometries.Add(geom.Hash); + tileHashes.Add(geom.Hash); } var file = $"{tilesetSettings.OutputSettings.ContentFolder}{Path.AltDirectorySeparatorChar}{tile.Level}_{tile.Z}_{tile.X}_{tile.Y}.glb"; From 7748bd4d68880193a99801a5bfdc2da8d46c72fe Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 11:05:26 +0100 Subject: [PATCH 37/57] Update src/b3dm.tileset/SpatialIndexChecker.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/b3dm.tileset/SpatialIndexChecker.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/b3dm.tileset/SpatialIndexChecker.cs b/src/b3dm.tileset/SpatialIndexChecker.cs index 21bd650e..0c682f85 100644 --- a/src/b3dm.tileset/SpatialIndexChecker.cs +++ b/src/b3dm.tileset/SpatialIndexChecker.cs @@ -10,7 +10,7 @@ public static bool HasSpatialIndex(NpgsqlConnection conn, string geometry_table, return HasIndex(conn, geometry_table, indexDef); } - public static bool HashMd5Index(NpgsqlConnection conn, string geometry_table, string geometry_column) + public static bool HasMd5Index(NpgsqlConnection conn, string geometry_table, string geometry_column) { var indexDef = $"%md5((st_asbinary({geometry_column}))::text%)"; return HasIndex(conn, geometry_table, indexDef); From bf838b7b1b3553376ad7f8cb2e1c38742144343e Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 11:06:29 +0100 Subject: [PATCH 38/57] Update src/b3dm.tileset/QuadtreeTiler.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/b3dm.tileset/QuadtreeTiler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index b04fe6a3..c8278c4c 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -133,7 +133,7 @@ private HashSet CreateTileForLargestGeometries(BoundingBox bbox, Tile ti ProcessLodLevels(bbox, tile, lod, createGltf, keepProjection); // todo: check the updateTileBoundingBox - if(!useImplicitTiling) { + if (!useImplicitTiling) { UpdateTileBoundingBox(tile, tileHashes, where, keepProjection); } From f893dc00c2cb66836dd93cfc847b9e10cd72a242 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 11:06:56 +0100 Subject: [PATCH 39/57] Update src/b3dm.tileset/QuadtreeTiler.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/b3dm.tileset/QuadtreeTiler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index c8278c4c..95a0c6cb 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -25,7 +25,7 @@ public class QuadtreeTiler private InputTable inputTable; private readonly bool useImplicitTiling; - public QuadtreeTiler(string connectionString, InputTable inputTable, StylingSettings stylingSettings, int maxFeaturesPerTile, double[] translation, string outputFolder, List lods, string copyright = "", bool skipCreateTiles = false, bool useImplicitTiling=false) + public QuadtreeTiler(string connectionString, InputTable inputTable, StylingSettings stylingSettings, int maxFeaturesPerTile, double[] translation, string outputFolder, List lods, string copyright = "", bool skipCreateTiles = false, bool useImplicitTiling = false) { this.conn = new NpgsqlConnection(connectionString); this.inputTable = inputTable; From 114543659eceb7c874b95c56f4dbd6488c5ce346 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 11:07:47 +0100 Subject: [PATCH 40/57] Update src/b3dm.tileset/OctreeTiler.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/b3dm.tileset/OctreeTiler.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/b3dm.tileset/OctreeTiler.cs b/src/b3dm.tileset/OctreeTiler.cs index 11411c03..2f0f41fc 100644 --- a/src/b3dm.tileset/OctreeTiler.cs +++ b/src/b3dm.tileset/OctreeTiler.cs @@ -135,10 +135,8 @@ private void CreateTile3D(BoundingBox3D bbox, int level, Tile3D tile, List 0) { // Collect hashes of processed geometries - foreach (var geom in geometries) { - if (!string.IsNullOrEmpty(geom.Hash)) { - processedGeometries.Add(geom.Hash); - } + foreach (var geom in geometries.Where(geom => !string.IsNullOrEmpty(geom.Hash))) { + processedGeometries.Add(geom.Hash); } var file = $"{tilesetSettings.OutputSettings.ContentFolder}{Path.AltDirectorySeparatorChar}{tile.Level}_{tile.Z}_{tile.X}_{tile.Y}.glb"; From 1dc997c4421766048a0cde7269d93f8930fe9fc7 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 11:47:47 +0100 Subject: [PATCH 41/57] fix typo --- src/pg2b3dm/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg2b3dm/Program.cs b/src/pg2b3dm/Program.cs index 45591c42..3b48c185 100644 --- a/src/pg2b3dm/Program.cs +++ b/src/pg2b3dm/Program.cs @@ -105,7 +105,7 @@ static void Main(string[] args) } // Check md5 index - var hasMd5Index = SpatialIndexChecker.HashMd5Index(conn, inputTable.TableName, inputTable.GeometryColumn); + var hasMd5Index = SpatialIndexChecker.HasMd5Index(conn, inputTable.TableName, inputTable.GeometryColumn); if (!hasMd5Index) { Console.WriteLine(); Console.WriteLine("-----------------------------------------------------------------------------"); From ea33e061dd31a2a38dce51f373eb10b975ae7983 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 10:48:52 +0000 Subject: [PATCH 42/57] Initial plan From 9b0a0cb381a2016867daea9779f5506d7e24160e Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 11:49:40 +0100 Subject: [PATCH 43/57] remove todo --- src/b3dm.tileset/QuadtreeTiler.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index 95a0c6cb..7206f8ab 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -132,7 +132,6 @@ private HashSet CreateTileForLargestGeometries(BoundingBox bbox, Tile ti TileCreationHelper.WriteTileIfNeeded(geometriesToProcess, translation, stylingSettings, copyright, createGltf, skipCreateTiles, outputPath, file); ProcessLodLevels(bbox, tile, lod, createGltf, keepProjection); - // todo: check the updateTileBoundingBox if (!useImplicitTiling) { UpdateTileBoundingBox(tile, tileHashes, where, keepProjection); } From 03c88655e1ad4fb6174484900fffeb119a085858 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 11:52:33 +0100 Subject: [PATCH 44/57] update readme --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 72d2b0de..3a7cc0ed 100644 --- a/README.md +++ b/README.md @@ -220,10 +220,11 @@ For styling see [styling 3D Tiles](styling.md) Input geometries must be of type LineString/MultilineString/Polygon/MultiPolygon/PolyhedralSurface (with z values). When the geometry is not triangulated, pg2b3dm will perform triangulation. Geometries with interior rings are supported. -For large datasets create a spatial index on the geometry column: +For large datasets create the following indexes: ``` -psql> CREATE INDEX ON the_table USING gist (st_envelope(geom)); +psql> CREATE INDEX ON the_table USING gist (st_centroid(st_envelope(geom))); +psql> CREATE INDEX ON the_table using btree(md5(st_asbinary({inputTable.GeometryColumn})::text)); ``` When there the spatial index is not present the following warning is shown. From 4ea012d444e7e4f69b58395405f73ee5faabc009 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 10:54:25 +0000 Subject: [PATCH 45/57] Pass processedGeometries to LOD child tiles to maintain deduplication Co-authored-by: bertt <538812+bertt@users.noreply.github.com> --- src/b3dm.tileset/QuadtreeTiler.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index 95a0c6cb..a7e07fdc 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -131,7 +131,7 @@ private HashSet CreateTileForLargestGeometries(BoundingBox bbox, Tile ti var outputPath = $"{outputFolder}{Path.AltDirectorySeparatorChar}{file}"; TileCreationHelper.WriteTileIfNeeded(geometriesToProcess, translation, stylingSettings, copyright, createGltf, skipCreateTiles, outputPath, file); - ProcessLodLevels(bbox, tile, lod, createGltf, keepProjection); + ProcessLodLevels(bbox, tile, lod, createGltf, keepProjection, localProcessedGeometries); // todo: check the updateTileBoundingBox if (!useImplicitTiling) { UpdateTileBoundingBox(tile, tileHashes, where, keepProjection); @@ -175,7 +175,7 @@ private void CreateTile(BoundingBox bbox, Tile tile, List tiles, string wh var outputPath = $"{outputFolder}{Path.AltDirectorySeparatorChar}{file}"; TileCreationHelper.WriteTileIfNeeded(geometries, translation, stylingSettings, copyright, createGltf, skipCreateTiles, outputPath, file); - ProcessLodLevels(bbox, tile, lod, createGltf, keepProjection); + ProcessLodLevels(bbox, tile, lod, createGltf, keepProjection, processedGeometries); if (!useImplicitTiling) { UpdateTileBoundingBox(tile, tileHashes, where, keepProjection); } @@ -188,7 +188,7 @@ private void CreateTile(BoundingBox bbox, Tile tile, List tiles, string wh tiles.Add(tile); } - private void ProcessLodLevels(BoundingBox bbox, Tile tile, int lod, bool createGltf, bool keepProjection) + private void ProcessLodLevels(BoundingBox bbox, Tile tile, int lod, bool createGltf, bool keepProjection, HashSet processedGeometries) { if (inputTable.LodColumn != String.Empty && lod < lods.Max()) { // take the next lod @@ -198,7 +198,7 @@ private void ProcessLodLevels(BoundingBox bbox, Tile tile, int lod, bool createG // make a copy of the tile var t2 = new Tile(tile.Z, tile.X, tile.Y); t2.BoundingBox = tile.BoundingBox; - var lodNextTiles = GenerateTiles(bbox, t2, new List(), nextLod, createGltf, keepProjection); + var lodNextTiles = GenerateTiles(bbox, t2, new List(), nextLod, createGltf, keepProjection, processedGeometries); tile.Children = lodNextTiles; } } From 48aa3076e7a4193757709897448f5d9eb69ff0cc Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 12:01:54 +0100 Subject: [PATCH 46/57] Update QuadtreeTiler.cs --- src/b3dm.tileset/QuadtreeTiler.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index 64897d8b..1862d115 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -131,12 +131,7 @@ private HashSet CreateTileForLargestGeometries(BoundingBox bbox, Tile ti var outputPath = $"{outputFolder}{Path.AltDirectorySeparatorChar}{file}"; TileCreationHelper.WriteTileIfNeeded(geometriesToProcess, translation, stylingSettings, copyright, createGltf, skipCreateTiles, outputPath, file); -<<<<<<< copilot/sub-pr-244-again ProcessLodLevels(bbox, tile, lod, createGltf, keepProjection, localProcessedGeometries); - // todo: check the updateTileBoundingBox -======= - ProcessLodLevels(bbox, tile, lod, createGltf, keepProjection); ->>>>>>> md5_implementation if (!useImplicitTiling) { UpdateTileBoundingBox(tile, tileHashes, where, keepProjection); } From f141ec7890c4dc0b0db19cfa611365c30d7e28de Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 12:54:44 +0100 Subject: [PATCH 47/57] remove octree tilehashes --- src/b3dm.tileset/OctreeTiler.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/b3dm.tileset/OctreeTiler.cs b/src/b3dm.tileset/OctreeTiler.cs index 2f0f41fc..4004b995 100644 --- a/src/b3dm.tileset/OctreeTiler.cs +++ b/src/b3dm.tileset/OctreeTiler.cs @@ -98,7 +98,6 @@ private HashSet CreateTileForLargestGeometries3D(BoundingBox3D bbox, int { // clone processedGeometries to avoid modifying the original set in recursive calls var localProcessedGeometries = new HashSet(processedGeometries); - var tileHashes = new HashSet(); // Get the largest geometries (up to MaxFeaturesPerTile) for this tile at this level int target_srs = tilingSettings.KeepProjection ? inputTable.EPSGCode : 4978; @@ -109,7 +108,6 @@ private HashSet CreateTileForLargestGeometries3D(BoundingBox3D bbox, int if (geometriesToProcess.Count > 0) { foreach (var geom in geometriesToProcess.Where(geom => !string.IsNullOrEmpty(geom.Hash))) { localProcessedGeometries.Add(geom.Hash); - tileHashes.Add(geom.Hash); } var file = $"{tilesetSettings.OutputSettings.ContentFolder}{Path.AltDirectorySeparatorChar}{tile.Level}_{tile.Z}_{tile.X}_{tile.Y}.glb"; From cd0c4bf0e0471a872f4f33997b79daf63fd06511 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 12:56:37 +0100 Subject: [PATCH 48/57] update readme --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3a7cc0ed..69713875 100644 --- a/README.md +++ b/README.md @@ -265,7 +265,11 @@ When the input geometries are distributed in a flat area (like buildings in a ci OCTREE is used when the input geometries are distributed in a cube-like area. -Most features are supported when using OCTREE subdivision, except LOD support; +Most features are supported when using OCTREE subdivision, except + +- LOD support; + +- Update boundingboxes when using explicit tiling; ## Query parameter From ce0434f10b7df6847872e55e14b9259f01d6f029 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 13:25:30 +0100 Subject: [PATCH 49/57] Update src/b3dm.tileset/SpatialIndexChecker.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/b3dm.tileset/SpatialIndexChecker.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/b3dm.tileset/SpatialIndexChecker.cs b/src/b3dm.tileset/SpatialIndexChecker.cs index 0c682f85..4598f99e 100644 --- a/src/b3dm.tileset/SpatialIndexChecker.cs +++ b/src/b3dm.tileset/SpatialIndexChecker.cs @@ -12,7 +12,7 @@ public static bool HasSpatialIndex(NpgsqlConnection conn, string geometry_table, public static bool HasMd5Index(NpgsqlConnection conn, string geometry_table, string geometry_column) { - var indexDef = $"%md5((st_asbinary({geometry_column}))::text%)"; + var indexDef = $"%md5(st_asbinary({geometry_column})::text)%"; return HasIndex(conn, geometry_table, indexDef); } From cbf15220e994e833d5f063265c494ddd71348978 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 13:27:59 +0100 Subject: [PATCH 50/57] Update README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3a7cc0ed..300c0270 100644 --- a/README.md +++ b/README.md @@ -224,7 +224,7 @@ For large datasets create the following indexes: ``` psql> CREATE INDEX ON the_table USING gist (st_centroid(st_envelope(geom))); -psql> CREATE INDEX ON the_table using btree(md5(st_asbinary({inputTable.GeometryColumn})::text)); +psql> CREATE INDEX ON the_table using btree(md5(st_asbinary(geom)::text)); ``` When there the spatial index is not present the following warning is shown. From 028a19c2d3b4b8f58c187d6a2d56add922efc671 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 5 Feb 2026 13:30:08 +0100 Subject: [PATCH 51/57] improve error handling --- src/b3dm.tileset/QuadtreeTiler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index 1862d115..6085c7ef 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -175,7 +175,7 @@ private void CreateTile(BoundingBox bbox, Tile tile, List tiles, string wh TileCreationHelper.WriteTileIfNeeded(geometries, translation, stylingSettings, copyright, createGltf, skipCreateTiles, outputPath, file); ProcessLodLevels(bbox, tile, lod, createGltf, keepProjection, processedGeometries); - if (!useImplicitTiling) { + if (!useImplicitTiling && tileHashes.Count > 0) { UpdateTileBoundingBox(tile, tileHashes, where, keepProjection); } From d5495a44f4b24da5496178d819033f8d52770026 Mon Sep 17 00:00:00 2001 From: bert Date: Mon, 9 Feb 2026 15:52:23 +0100 Subject: [PATCH 52/57] improve md5 hash index check --- src/b3dm.tileset/SpatialIndexChecker.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/b3dm.tileset/SpatialIndexChecker.cs b/src/b3dm.tileset/SpatialIndexChecker.cs index 4598f99e..eec6d978 100644 --- a/src/b3dm.tileset/SpatialIndexChecker.cs +++ b/src/b3dm.tileset/SpatialIndexChecker.cs @@ -12,7 +12,7 @@ public static bool HasSpatialIndex(NpgsqlConnection conn, string geometry_table, public static bool HasMd5Index(NpgsqlConnection conn, string geometry_table, string geometry_column) { - var indexDef = $"%md5(st_asbinary({geometry_column})::text)%"; + var indexDef = $"%md5((st_asbinary({geometry_column}))::text)%"; return HasIndex(conn, geometry_table, indexDef); } From 22b9fe3604146ff6a95a33154d0d9e8b53b1d8e2 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Tue, 10 Feb 2026 12:21:35 +0100 Subject: [PATCH 53/57] update solution --- src/pg2b3dm.sln | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg2b3dm.sln b/src/pg2b3dm.sln index 6f234a34..d1a1488a 100644 --- a/src/pg2b3dm.sln +++ b/src/pg2b3dm.sln @@ -25,7 +25,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "doc", "doc", "{0403ED2E-B5A ..\..\..\..\..\Users\bert\Desktop\demo_pg2b3dm.gif = ..\..\..\..\..\Users\bert\Desktop\demo_pg2b3dm.gif ..\getting_started.md = ..\getting_started.md ..\dataprocessing\materials_for_features.csv = ..\dataprocessing\materials_for_features.csv - md5_queries.md = md5_queries.md + ..\md5_queries.md = ..\md5_queries.md ..\README.md = ..\README.md release_notes_0.10.md = release_notes_0.10.md release_notes_0.11.md = release_notes_0.11.md From 03297e6ecc7c0e786f841bde87f09345b349f7b6 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Tue, 10 Feb 2026 12:53:56 +0100 Subject: [PATCH 54/57] update octreetiler --- src/b3dm.tileset/GeometryRepository.cs | 9 ++++++--- src/b3dm.tileset/OctreeTiler.cs | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/b3dm.tileset/GeometryRepository.cs b/src/b3dm.tileset/GeometryRepository.cs index 8b9443d6..127dbd0d 100644 --- a/src/b3dm.tileset/GeometryRepository.cs +++ b/src/b3dm.tileset/GeometryRepository.cs @@ -85,7 +85,7 @@ public static double[] GetGeometriesBoundingBox(NpgsqlConnection conn, string ge } } - public static List GetGeometrySubset(NpgsqlConnection conn, string geometry_table, string geometry_column, double[] bbox, int source_epsg, int target_srs, string shaderColumn = "", string attributesColumns = "", string query = "", string radiusColumn = "", bool keepProjection = false, HashSet excludeHashes = null, int? maxFeatures = null) + public static List GetGeometrySubset(NpgsqlConnection conn, string geometry_table, string geometry_column, double[] bbox, int source_epsg, int target_srs, string shaderColumn = "", string attributesColumns = "", string query = "", string radiusColumn = "", bool keepProjection = false, HashSet excludeHashes = null, int? maxFeatures = null, SubdivisionScheme scheme = SubdivisionScheme.QUADTREE) { var sqlselect = GetSqlSelect(geometry_column, shaderColumn, attributesColumns, radiusColumn, target_srs); var sqlFrom = "FROM " + geometry_table; @@ -100,7 +100,7 @@ public static List GetGeometrySubset(NpgsqlConnection conn, stri sqlWhere += $" AND MD5(ST_AsBinary({geometry_column})::text) != ALL(@excludeHashes)"; } - var sqlOrderBy = GetOrderBy(geometry_column); + var sqlOrderBy = GetOrderBy(geometry_column, scheme); var sqlLimit = maxFeatures.HasValue ? $" LIMIT {maxFeatures.Value}" : ""; var sql = sqlselect + sqlFrom + " where " + sqlWhere + sqlOrderBy + sqlLimit; @@ -175,8 +175,11 @@ public static string GetGeometryColumn(string geometry_column, int target_srs) return $"st_transform({geometry_column}, {target_srs})"; } - public static string GetOrderBy(string geometry_column) + public static string GetOrderBy(string geometry_column, SubdivisionScheme scheme) { + if (scheme == SubdivisionScheme.OCTREE) { + return $" ORDER BY (ST_XMax({geometry_column}) - ST_XMin({geometry_column})) *(ST_YMax({geometry_column}) - ST_YMin({geometry_column})) DESC"; + } return $" ORDER BY ST_Area(ST_Envelope({geometry_column})) DESC"; } diff --git a/src/b3dm.tileset/OctreeTiler.cs b/src/b3dm.tileset/OctreeTiler.cs index 4004b995..f38653d6 100644 --- a/src/b3dm.tileset/OctreeTiler.cs +++ b/src/b3dm.tileset/OctreeTiler.cs @@ -103,7 +103,7 @@ private HashSet CreateTileForLargestGeometries3D(BoundingBox3D bbox, int int target_srs = tilingSettings.KeepProjection ? inputTable.EPSGCode : 4978; var bbox1 = new double[] { bbox.XMin, bbox.YMin, bbox.XMax, bbox.YMax, bbox.ZMin, bbox.ZMax }; - var geometriesToProcess = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, bbox1, inputTable.EPSGCode, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, tilingSettings.KeepProjection, processedGeometries, tilingSettings.MaxFeaturesPerTile); + var geometriesToProcess = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, bbox1, inputTable.EPSGCode, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, tilingSettings.KeepProjection, processedGeometries, tilingSettings.MaxFeaturesPerTile, SubdivisionScheme.OCTREE); if (geometriesToProcess.Count > 0) { foreach (var geom in geometriesToProcess.Where(geom => !string.IsNullOrEmpty(geom.Hash))) { @@ -129,7 +129,7 @@ private void CreateTile3D(BoundingBox3D bbox, int level, Tile3D tile, List 0) { // Collect hashes of processed geometries From 5a6b4b42a2a5e340c09ace35d3c888fb8c972181 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Tue, 10 Feb 2026 20:34:15 +0100 Subject: [PATCH 55/57] remove todo --- src/b3dm.tileset/GeometryRepository.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/b3dm.tileset/GeometryRepository.cs b/src/b3dm.tileset/GeometryRepository.cs index 127dbd0d..25184614 100644 --- a/src/b3dm.tileset/GeometryRepository.cs +++ b/src/b3dm.tileset/GeometryRepository.cs @@ -89,8 +89,6 @@ public static List GetGeometrySubset(NpgsqlConnection conn, stri { var sqlselect = GetSqlSelect(geometry_column, shaderColumn, attributesColumns, radiusColumn, target_srs); var sqlFrom = "FROM " + geometry_table; - - // todo: fix unit test when there is no z var points = GetPoints(bbox); var sqlWhere = GetWhere(geometry_column, points.fromPoint, points.toPoint, query, source_epsg, keepProjection); From 45c749dbd99a3d2b81847b7adfb70e339474cf42 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Thu, 12 Feb 2026 10:59:40 +0100 Subject: [PATCH 56/57] add option sortby - area or volume (default area) --- README.md | 6 ++++ src/b3dm.tileset.tests/CesiumTilerTests.cs | 1 - .../GeometryRepositoryOrderByTests.cs | 29 +++++++++++++++++++ src/b3dm.tileset/CesiumTiler.cs | 1 - src/b3dm.tileset/GeometryRepository.cs | 10 +++---- src/b3dm.tileset/OctreeTiler.cs | 10 +++---- src/b3dm.tileset/OutputDirectoryCreator.cs | 2 +- src/b3dm.tileset/QuadtreeTiler.cs | 9 +++--- src/b3dm.tileset/TileCreationHelper.cs | 1 - src/b3dm.tileset/settings/InputTable.cs | 2 +- src/b3dm.tileset/settings/OutputSettings.cs | 2 +- src/b3dm.tileset/settings/SortBy.cs | 11 +++++++ src/b3dm.tileset/settings/StylingSettings.cs | 2 +- src/b3dm.tileset/settings/TilesetSettings.cs | 2 +- src/b3dm.tileset/settings/TilingSettings.cs | 4 ++- src/pg2b3dm.database.tests/UnitTest1.cs | 1 - src/pg2b3dm/Options.cs | 3 ++ src/pg2b3dm/Program.cs | 5 ++-- 18 files changed, 74 insertions(+), 27 deletions(-) create mode 100644 src/b3dm.tileset.tests/GeometryRepositoryOrderByTests.cs create mode 100644 src/b3dm.tileset/settings/SortBy.cs diff --git a/README.md b/README.md index 0d20a15d..02c45d17 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,12 @@ If --username and/or --dbname are not specified the current username is used as --keep_projection (Default: false) Keep projection of input data + --sortby (Default: AREA) Sort features by AREA/VOLUME + AREA is the default and faster (uses ST_Area). + VOLUME is slower (uses 3D bounding-box volume) and is useful when + geometries have relatively large Z components (e.g. infrastructure / + vertical walls). + --subdivision (Default: QUADTREE) Subdivision schema QUADTREE/OCTREE --help Display this help screen. diff --git a/src/b3dm.tileset.tests/CesiumTilerTests.cs b/src/b3dm.tileset.tests/CesiumTilerTests.cs index 72a80782..c9cfd923 100644 --- a/src/b3dm.tileset.tests/CesiumTilerTests.cs +++ b/src/b3dm.tileset.tests/CesiumTilerTests.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using B3dm.Tileset.settings; using NUnit.Framework; using pg2b3dm; using subtree; diff --git a/src/b3dm.tileset.tests/GeometryRepositoryOrderByTests.cs b/src/b3dm.tileset.tests/GeometryRepositoryOrderByTests.cs new file mode 100644 index 00000000..1f2b973e --- /dev/null +++ b/src/b3dm.tileset.tests/GeometryRepositoryOrderByTests.cs @@ -0,0 +1,29 @@ +using NUnit.Framework; + +namespace B3dm.Tileset.Tests; + +public class GeometryRepositoryOrderByTests +{ + [Test] + public void GetOrderBy_Area_UsesEnvelopeArea() + { + var sql = GeometryRepository.GetOrderBy("geom", SortBy.AREA); + Assert.That(sql, Does.Contain("ST_Area(ST_Envelope(geom))")); + Assert.That(sql, Does.Contain("DESC")); + } + + [Test] + public void GetOrderBy_Volume_UsesZExtents() + { + var sql = GeometryRepository.GetOrderBy("geom", SortBy.VOLUME); + Assert.That(sql, Does.Contain("ST_ZMax(geom)")); + Assert.That(sql, Does.Contain("ST_ZMin(geom)")); + } + + [Test] + public void TilingSettings_DefaultSortBy_IsArea() + { + var settings = new TilingSettings(); + Assert.That(settings.SortBy, Is.EqualTo(SortBy.AREA)); + } +} diff --git a/src/b3dm.tileset/CesiumTiler.cs b/src/b3dm.tileset/CesiumTiler.cs index 53f1c232..d5a5d964 100644 --- a/src/b3dm.tileset/CesiumTiler.cs +++ b/src/b3dm.tileset/CesiumTiler.cs @@ -3,7 +3,6 @@ using System.IO; using System.Linq; using B3dm.Tileset.Extensions; -using B3dm.Tileset.settings; using Newtonsoft.Json; using subtree; using Wkx; diff --git a/src/b3dm.tileset/GeometryRepository.cs b/src/b3dm.tileset/GeometryRepository.cs index 25184614..bac1735b 100644 --- a/src/b3dm.tileset/GeometryRepository.cs +++ b/src/b3dm.tileset/GeometryRepository.cs @@ -85,7 +85,7 @@ public static double[] GetGeometriesBoundingBox(NpgsqlConnection conn, string ge } } - public static List GetGeometrySubset(NpgsqlConnection conn, string geometry_table, string geometry_column, double[] bbox, int source_epsg, int target_srs, string shaderColumn = "", string attributesColumns = "", string query = "", string radiusColumn = "", bool keepProjection = false, HashSet excludeHashes = null, int? maxFeatures = null, SubdivisionScheme scheme = SubdivisionScheme.QUADTREE) + public static List GetGeometrySubset(NpgsqlConnection conn, string geometry_table, string geometry_column, double[] bbox, int source_epsg, int target_srs, string shaderColumn = "", string attributesColumns = "", string query = "", string radiusColumn = "", bool keepProjection = false, HashSet excludeHashes = null, int? maxFeatures = null, SortBy sortBy = SortBy.AREA) { var sqlselect = GetSqlSelect(geometry_column, shaderColumn, attributesColumns, radiusColumn, target_srs); var sqlFrom = "FROM " + geometry_table; @@ -98,7 +98,7 @@ public static List GetGeometrySubset(NpgsqlConnection conn, stri sqlWhere += $" AND MD5(ST_AsBinary({geometry_column})::text) != ALL(@excludeHashes)"; } - var sqlOrderBy = GetOrderBy(geometry_column, scheme); + var sqlOrderBy = GetOrderBy(geometry_column, sortBy); var sqlLimit = maxFeatures.HasValue ? $" LIMIT {maxFeatures.Value}" : ""; var sql = sqlselect + sqlFrom + " where " + sqlWhere + sqlOrderBy + sqlLimit; @@ -173,10 +173,10 @@ public static string GetGeometryColumn(string geometry_column, int target_srs) return $"st_transform({geometry_column}, {target_srs})"; } - public static string GetOrderBy(string geometry_column, SubdivisionScheme scheme) + public static string GetOrderBy(string geometry_column, SortBy sortBy) { - if (scheme == SubdivisionScheme.OCTREE) { - return $" ORDER BY (ST_XMax({geometry_column}) - ST_XMin({geometry_column})) *(ST_YMax({geometry_column}) - ST_YMin({geometry_column})) DESC"; + if (sortBy == SortBy.VOLUME) { + return $" ORDER BY (ST_XMax({geometry_column}) - ST_XMin({geometry_column})) *(ST_YMax({geometry_column}) - ST_YMin({geometry_column})) *(ST_ZMax({geometry_column}) - ST_ZMin({geometry_column})) DESC"; } return $" ORDER BY ST_Area(ST_Envelope({geometry_column})) DESC"; } diff --git a/src/b3dm.tileset/OctreeTiler.cs b/src/b3dm.tileset/OctreeTiler.cs index f38653d6..1d97882a 100644 --- a/src/b3dm.tileset/OctreeTiler.cs +++ b/src/b3dm.tileset/OctreeTiler.cs @@ -1,10 +1,8 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.IO; using System.Linq; -using B3dm.Tileset.settings; +using B3dm.Tileset; using Npgsql; -using pg2b3dm; using subtree; using Wkx; @@ -103,7 +101,7 @@ private HashSet CreateTileForLargestGeometries3D(BoundingBox3D bbox, int int target_srs = tilingSettings.KeepProjection ? inputTable.EPSGCode : 4978; var bbox1 = new double[] { bbox.XMin, bbox.YMin, bbox.XMax, bbox.YMax, bbox.ZMin, bbox.ZMax }; - var geometriesToProcess = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, bbox1, inputTable.EPSGCode, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, tilingSettings.KeepProjection, processedGeometries, tilingSettings.MaxFeaturesPerTile, SubdivisionScheme.OCTREE); + var geometriesToProcess = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, bbox1, inputTable.EPSGCode, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, tilingSettings.KeepProjection, processedGeometries, tilingSettings.MaxFeaturesPerTile, tilingSettings.SortBy); if (geometriesToProcess.Count > 0) { foreach (var geom in geometriesToProcess.Where(geom => !string.IsNullOrEmpty(geom.Hash))) { @@ -129,7 +127,7 @@ private void CreateTile3D(BoundingBox3D bbox, int level, Tile3D tile, List 0) { // Collect hashes of processed geometries diff --git a/src/b3dm.tileset/OutputDirectoryCreator.cs b/src/b3dm.tileset/OutputDirectoryCreator.cs index 8f1c8f69..998b9a10 100644 --- a/src/b3dm.tileset/OutputDirectoryCreator.cs +++ b/src/b3dm.tileset/OutputDirectoryCreator.cs @@ -1,5 +1,5 @@ using System.IO; -using B3dm.Tileset.settings; +using B3dm.Tileset; namespace pg2b3dm; diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index 6085c7ef..45d70c98 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -4,7 +4,6 @@ using System.Linq; using B3dm.Tileset; using B3dm.Tileset.Extensions; -using B3dm.Tileset.settings; using Npgsql; using subtree; using Wkx; @@ -24,8 +23,9 @@ public class QuadtreeTiler private readonly StylingSettings stylingSettings; private InputTable inputTable; private readonly bool useImplicitTiling; + private readonly SortBy sortBy; - public QuadtreeTiler(string connectionString, InputTable inputTable, StylingSettings stylingSettings, int maxFeaturesPerTile, double[] translation, string outputFolder, List lods, string copyright = "", bool skipCreateTiles = false, bool useImplicitTiling = false) + public QuadtreeTiler(string connectionString, InputTable inputTable, StylingSettings stylingSettings, int maxFeaturesPerTile, double[] translation, string outputFolder, List lods, string copyright = "", bool skipCreateTiles = false, bool useImplicitTiling = false, SortBy sortBy = SortBy.AREA) { this.conn = new NpgsqlConnection(connectionString); this.inputTable = inputTable; @@ -38,6 +38,7 @@ public QuadtreeTiler(string connectionString, InputTable inputTable, StylingSett this.skipCreateTiles = skipCreateTiles; this.stylingSettings = stylingSettings; this.useImplicitTiling = useImplicitTiling; + this.sortBy = sortBy; } public List GenerateTiles(BoundingBox bbox, Tile tile, List tiles, int lod = 0, bool createGltf = false, bool keepProjection = false, HashSet processedGeometries = null) @@ -106,7 +107,7 @@ private HashSet CreateTileForLargestGeometries(BoundingBox bbox, Tile ti int target_srs = keepProjection ? source_epsg : 4978; - var geometriesToProcess = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, tile.BoundingBox, source_epsg, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, keepProjection, processedGeometries, maxFeaturesPerTile); + var geometriesToProcess = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, tile.BoundingBox, source_epsg, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, keepProjection, processedGeometries, maxFeaturesPerTile, sortBy: sortBy); if (geometriesToProcess.Count > 0) { @@ -160,7 +161,7 @@ private void CreateTile(BoundingBox bbox, Tile tile, List tiles, string wh int target_srs = keepProjection ? source_epsg : 4978; - var geometries = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, tile.BoundingBox, source_epsg, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, keepProjection, processedGeometries); + var geometries = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, tile.BoundingBox, source_epsg, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, keepProjection, processedGeometries, sortBy: sortBy); if (geometries.Count > 0) { // Collect hashes of processed geometries diff --git a/src/b3dm.tileset/TileCreationHelper.cs b/src/b3dm.tileset/TileCreationHelper.cs index aa209a7a..de2c59d4 100644 --- a/src/b3dm.tileset/TileCreationHelper.cs +++ b/src/b3dm.tileset/TileCreationHelper.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using B3dm.Tileset.settings; using pg2b3dm; using Wkb2Gltf; diff --git a/src/b3dm.tileset/settings/InputTable.cs b/src/b3dm.tileset/settings/InputTable.cs index 98aa45e1..f69d953b 100644 --- a/src/b3dm.tileset/settings/InputTable.cs +++ b/src/b3dm.tileset/settings/InputTable.cs @@ -1,4 +1,4 @@ -namespace B3dm.Tileset.settings; +namespace B3dm.Tileset; public class InputTable { diff --git a/src/b3dm.tileset/settings/OutputSettings.cs b/src/b3dm.tileset/settings/OutputSettings.cs index 3b5bb00a..c4bb224b 100644 --- a/src/b3dm.tileset/settings/OutputSettings.cs +++ b/src/b3dm.tileset/settings/OutputSettings.cs @@ -1,4 +1,4 @@ -namespace B3dm.Tileset.settings; +namespace B3dm.Tileset; public class OutputSettings { diff --git a/src/b3dm.tileset/settings/SortBy.cs b/src/b3dm.tileset/settings/SortBy.cs new file mode 100644 index 00000000..c39f187d --- /dev/null +++ b/src/b3dm.tileset/settings/SortBy.cs @@ -0,0 +1,11 @@ +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace B3dm.Tileset; + +[JsonConverter(typeof(StringEnumConverter))] +public enum SortBy +{ + AREA, + VOLUME +} diff --git a/src/b3dm.tileset/settings/StylingSettings.cs b/src/b3dm.tileset/settings/StylingSettings.cs index 25747b0d..1d24108f 100644 --- a/src/b3dm.tileset/settings/StylingSettings.cs +++ b/src/b3dm.tileset/settings/StylingSettings.cs @@ -1,6 +1,6 @@ using SharpGLTF.Materials; -namespace B3dm.Tileset.settings; +namespace B3dm.Tileset; public class StylingSettings { diff --git a/src/b3dm.tileset/settings/TilesetSettings.cs b/src/b3dm.tileset/settings/TilesetSettings.cs index 71fb39ed..f7f4d0b8 100644 --- a/src/b3dm.tileset/settings/TilesetSettings.cs +++ b/src/b3dm.tileset/settings/TilesetSettings.cs @@ -1,6 +1,6 @@ using System; -namespace B3dm.Tileset.settings; +namespace B3dm.Tileset; public class TilesetSettings { diff --git a/src/b3dm.tileset/settings/TilingSettings.cs b/src/b3dm.tileset/settings/TilingSettings.cs index e4892981..b6e56671 100644 --- a/src/b3dm.tileset/settings/TilingSettings.cs +++ b/src/b3dm.tileset/settings/TilingSettings.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; -namespace B3dm.Tileset.settings; +namespace B3dm.Tileset; public class TilingSettings { @@ -14,6 +14,8 @@ public class TilingSettings public int MaxFeaturesPerTile { get; set; } = 1000; + public SortBy SortBy { get; set; } = SortBy.AREA; + public bool UseImplicitTiling { get; set; } = true; public List Lods { get; set; } diff --git a/src/pg2b3dm.database.tests/UnitTest1.cs b/src/pg2b3dm.database.tests/UnitTest1.cs index 1469d2d1..3a2ab4e7 100644 --- a/src/pg2b3dm.database.tests/UnitTest1.cs +++ b/src/pg2b3dm.database.tests/UnitTest1.cs @@ -1,6 +1,5 @@ using B3dm.Tileset; using B3dm.Tileset.Extensions; -using B3dm.Tileset.settings; using DotNet.Testcontainers.Builders; using Npgsql; using subtree; diff --git a/src/pg2b3dm/Options.cs b/src/pg2b3dm/Options.cs index 47c279e6..64cdf313 100644 --- a/src/pg2b3dm/Options.cs +++ b/src/pg2b3dm/Options.cs @@ -83,6 +83,9 @@ public class Options [Option("keep_projection", Required = false, Default = false, HelpText = "Keep projection")] public bool? KeepProjection { get; set; } + [Option("sortby", Required = false, Default = SortBy.AREA, HelpText = "Sort features by AREA (default) or VOLUME")] + public SortBy SortBy { get; set; } + [Option("subdivision", Required=false, Default=SubdivisionScheme.QUADTREE, HelpText = "Subdivision schema QUADTREE/OCTREE" )] public SubdivisionScheme subdivisionScheme { get; set; } } diff --git a/src/pg2b3dm/Program.cs b/src/pg2b3dm/Program.cs index 3b48c185..817111a1 100644 --- a/src/pg2b3dm/Program.cs +++ b/src/pg2b3dm/Program.cs @@ -9,7 +9,6 @@ using subtree; using B3dm.Tileset.Extensions; using SharpGLTF.Schema2; -using B3dm.Tileset.settings; namespace pg2b3dm; @@ -175,6 +174,7 @@ static void Main(string[] args) Console.WriteLine($"Refinement: {refinement}"); Console.WriteLine($"Keep projection: {keepProjection}"); Console.WriteLine($"Subdivision scheme: {subdivisionScheme}"); + Console.WriteLine($"Sort by: {o.SortBy}"); if (keepProjection && !useImplicitTiling) { Console.WriteLine("Warning: keepProjection is only supported with implicit tiling."); @@ -237,6 +237,7 @@ static void Main(string[] args) tilingSettings.UseImplicitTiling = useImplicitTiling; tilingSettings.SkipCreateTiles = skipCreateTiles; tilingSettings.MaxFeaturesPerTile = maxFeaturesPerTile; + tilingSettings.SortBy = o.SortBy; tilingSettings.Lods = lods; if (subdivisionScheme == SubdivisionScheme.QUADTREE) { @@ -293,7 +294,7 @@ private static void QuadtreeTile(string connectionString, InputTable inputTable, tile.BoundingBox = bbox.ToArray(); var outputSettings = tilesetSettings.OutputSettings; - var quadtreeTiler = new QuadtreeTiler(connectionString, inputTable, stylingSettings, tilingSettings.MaxFeaturesPerTile, tilesetSettings.Translation, outputSettings.ContentFolder, tilingSettings.Lods, tilesetSettings.Copyright, tilingSettings.SkipCreateTiles, tilingSettings.UseImplicitTiling); + var quadtreeTiler = new QuadtreeTiler(connectionString, inputTable, stylingSettings, tilingSettings.MaxFeaturesPerTile, tilesetSettings.Translation, outputSettings.ContentFolder, tilingSettings.Lods, tilesetSettings.Copyright, tilingSettings.SkipCreateTiles, tilingSettings.UseImplicitTiling, sortBy: tilingSettings.SortBy); var tiles = quadtreeTiler.GenerateTiles(bbox, tile, new List(), inputTable.LodColumn != string.Empty ? tilingSettings.Lods.First() : 0, tilingSettings.CreateGltf, tilingSettings.KeepProjection); Console.WriteLine(); Console.WriteLine("Tiles created: " + tiles.Count(tile => tile.Available)); From 87b3fd2b2070ec217546fd446564cc0a9e205468 Mon Sep 17 00:00:00 2001 From: Bert Temme Date: Wed, 25 Feb 2026 13:36:07 +0100 Subject: [PATCH 57/57] filter boundingboxes based on original projection (not epsg:4326) --- .../GeometryRepositoryWhereTests.cs | 61 ++++++++++++++++++ src/b3dm.tileset/BoundingBoxRepository.cs | 28 +++++++- src/b3dm.tileset/FeatureCountRepository.cs | 4 +- src/b3dm.tileset/GeometryRepository.cs | 64 ++----------------- src/b3dm.tileset/OctreeTiler.cs | 10 ++- src/b3dm.tileset/QuadtreeTiler.cs | 10 ++- src/pg2b3dm.database.tests/UnitTest1.cs | 7 +- src/pg2b3dm/Program.cs | 25 +++++--- 8 files changed, 127 insertions(+), 82 deletions(-) create mode 100644 src/b3dm.tileset.tests/GeometryRepositoryWhereTests.cs diff --git a/src/b3dm.tileset.tests/GeometryRepositoryWhereTests.cs b/src/b3dm.tileset.tests/GeometryRepositoryWhereTests.cs new file mode 100644 index 00000000..0213ace7 --- /dev/null +++ b/src/b3dm.tileset.tests/GeometryRepositoryWhereTests.cs @@ -0,0 +1,61 @@ +using NUnit.Framework; +using Wkx; + +namespace B3dm.Tileset.Tests; + +public class GeometryRepositoryWhereTests +{ + [Test] + public void GetWhere_2D_UseSourceEpsgDirectly() + { + var from = new Point(100000.0, 400000.0); + var to = new Point(200000.0, 500000.0); + + var sql = GeometryRepository.GetWhere("geom", from, to, "", 5698); + + Assert.That(sql, Does.Contain("5698")); + Assert.That(sql, Does.Not.Contain("4326")); + Assert.That(sql, Does.Not.Contain("ST_Transform")); + Assert.That(sql, Does.Contain("ST_MakeEnvelope")); + } + + [Test] + public void GetWhere_2D_Epsg4326_UseSourceEpsgDirectly() + { + var from = new Point(-75.8, 38.4); + var to = new Point(-75.0, 39.8); + + var sql = GeometryRepository.GetWhere("geom", from, to, "", 4326); + + Assert.That(sql, Does.Contain("4326")); + Assert.That(sql, Does.Not.Contain("ST_Transform")); + Assert.That(sql, Does.Contain("ST_MakeEnvelope")); + } + + [Test] + public void GetWhere_3D_UseSourceEpsgDirectly() + { + var from = new Point(100000.0, 400000.0, 200.0); + var to = new Point(200000.0, 500000.0, 300.0); + + var sql = GeometryRepository.GetWhere("geom", from, to, "", 5698); + + Assert.That(sql, Does.Contain("5698")); + Assert.That(sql, Does.Not.Contain("4979")); + Assert.That(sql, Does.Not.Contain("4326")); + Assert.That(sql, Does.Not.Contain("ST_Transform")); + Assert.That(sql, Does.Contain("ST_3DMakeBox")); + } + + [Test] + public void GetWhere_3D_ContainsCentroidAndSrid() + { + var from = new Point(100000.0, 400000.0, 200.0); + var to = new Point(200000.0, 500000.0, 300.0); + + var sql = GeometryRepository.GetWhere("geom", from, to, "", 5698); + + Assert.That(sql, Does.Contain("st_setsrid")); + Assert.That(sql, Does.Contain("ST_3DIntersects")); + } +} diff --git a/src/b3dm.tileset/BoundingBoxRepository.cs b/src/b3dm.tileset/BoundingBoxRepository.cs index e64b7e59..7a719bbc 100644 --- a/src/b3dm.tileset/BoundingBoxRepository.cs +++ b/src/b3dm.tileset/BoundingBoxRepository.cs @@ -1,4 +1,5 @@ -using System.Data; +using System; +using System.Data; using Wkx; namespace B3dm.Tileset; @@ -16,6 +17,31 @@ public static (BoundingBox bbox, double zmin, double zmax) GetBoundingBoxForTabl return bbox3d; } + public static (BoundingBox bbox, double zmin, double zmax) GetBoundingBoxAs4979(IDbConnection conn,(BoundingBox bbox, double zmin, double zmax) bboxTable, int sourceEpsg) + { + if (sourceEpsg == 4979 || sourceEpsg == 4326) { + return bboxTable; + } + var bbox = bboxTable.bbox; + var sqlBounds = FormattableString.Invariant($@"SELECT st_xmin(geom1),st_ymin(geom1), st_xmax(geom1), st_ymax(geom1), st_zmin(geom1), st_zmax(geom1) +FROM ( + SELECT ST_3DExtent( + ST_Transform( + ST_SetSRID( + ST_3DMakeBox( + ST_MakePoint({bbox.XMin}, {bbox.YMin}, {bboxTable.zmin}), + ST_MakePoint({bbox.XMax}, {bbox.YMax}, {bboxTable.zmax}) + ), + {sourceEpsg} + ), + 4979 + ) + ) AS geom1 +) AS t"); + return GetBounds(conn, sqlBounds); + } + + private static (BoundingBox, double, double) GetBounds(IDbConnection conn, string sql) { conn.Open(); diff --git a/src/b3dm.tileset/FeatureCountRepository.cs b/src/b3dm.tileset/FeatureCountRepository.cs index c21af284..c0f2e203 100644 --- a/src/b3dm.tileset/FeatureCountRepository.cs +++ b/src/b3dm.tileset/FeatureCountRepository.cs @@ -7,10 +7,10 @@ namespace B3dm.Tileset; public static class FeatureCountRepository { - public static int CountFeaturesInBox(NpgsqlConnection conn, string geometry_table, string geometry_column, Point from, Point to, string query, int source_epsg, bool keepProjection = false, HashSet excludeHashes = null) + public static int CountFeaturesInBox(NpgsqlConnection conn, string geometry_table, string geometry_column, Point from, Point to, string query, int source_epsg, HashSet excludeHashes = null) { var select = $"COUNT({geometry_column})"; - var where = GeometryRepository.GetWhere(geometry_column, from, to, query, source_epsg, keepProjection); + var where = GeometryRepository.GetWhere(geometry_column, from, to, query, source_epsg); // Add hash exclusion filter using parameterized query if (excludeHashes != null && excludeHashes.Count > 0) { diff --git a/src/b3dm.tileset/GeometryRepository.cs b/src/b3dm.tileset/GeometryRepository.cs index bac1735b..31029240 100644 --- a/src/b3dm.tileset/GeometryRepository.cs +++ b/src/b3dm.tileset/GeometryRepository.cs @@ -13,48 +13,6 @@ namespace B3dm.Tileset; public static class GeometryRepository { - public static HashSet FilterHashesByEnvelope(NpgsqlConnection conn, string tableName, string geometryColumn, BoundingBox bbox, int source_epsg, HashSet geometryHashes, bool keepProjection) - { - if (geometryHashes.Count == 0) { - return new HashSet(); - } - - var filteredHashes = new HashSet(); - - conn.Open(); - try { - var envelope = keepProjection ? - $"ST_MakeEnvelope(@xmin, @ymin, @xmax, @ymax, {source_epsg})" : - $"ST_Transform(ST_MakeEnvelope(@xmin, @ymin, @xmax, @ymax, 4326), {source_epsg})"; - - var query = $@" - SELECT MD5(ST_AsBinary({geometryColumn})::text) as geom_hash - FROM {tableName} - WHERE MD5(ST_AsBinary({geometryColumn})::text) = ANY(@hashes) - AND ST_Within( - ST_Centroid(ST_Envelope({geometryColumn})), - {envelope} - )"; - - using var cmd = new NpgsqlCommand(query, conn); - cmd.Parameters.AddWithValue("hashes", geometryHashes.ToArray()); - cmd.Parameters.AddWithValue("xmin", bbox.XMin); - cmd.Parameters.AddWithValue("ymin", bbox.YMin); - cmd.Parameters.AddWithValue("xmax", bbox.XMax); - cmd.Parameters.AddWithValue("ymax", bbox.YMax); - - using var reader = cmd.ExecuteReader(); - while (reader.Read()) { - var hash = reader.GetString(0); - filteredHashes.Add(hash); - } - - return filteredHashes; - } - finally { - conn.Close(); - } - } /// /// Returns double array with 6 bounding box coordinates, xmin, ymin, xmax, ymax, zmin, zmax @@ -85,13 +43,13 @@ public static double[] GetGeometriesBoundingBox(NpgsqlConnection conn, string ge } } - public static List GetGeometrySubset(NpgsqlConnection conn, string geometry_table, string geometry_column, double[] bbox, int source_epsg, int target_srs, string shaderColumn = "", string attributesColumns = "", string query = "", string radiusColumn = "", bool keepProjection = false, HashSet excludeHashes = null, int? maxFeatures = null, SortBy sortBy = SortBy.AREA) + public static List GetGeometrySubset(NpgsqlConnection conn, string geometry_table, string geometry_column, double[] bbox, int source_epsg, int target_srs, string shaderColumn = "", string attributesColumns = "", string query = "", string radiusColumn = "", HashSet excludeHashes = null, int? maxFeatures = null, SortBy sortBy = SortBy.AREA) { var sqlselect = GetSqlSelect(geometry_column, shaderColumn, attributesColumns, radiusColumn, target_srs); var sqlFrom = "FROM " + geometry_table; var points = GetPoints(bbox); - var sqlWhere = GetWhere(geometry_column, points.fromPoint, points.toPoint, query, source_epsg, keepProjection); + var sqlWhere = GetWhere(geometry_column, points.fromPoint, points.toPoint, query, source_epsg); // Add hash exclusion filter using parameterized query if (excludeHashes != null && excludeHashes.Count > 0) { @@ -117,7 +75,7 @@ public static List GetGeometrySubset(NpgsqlConnection conn, stri } } - public static string GetWhere(string geometry_column, Point from, Point to, string query, int source_epsg, bool keepProjection) + public static string GetWhere(string geometry_column, Point from, Point to, string query, int source_epsg) { var fromX = from.X.Value.ToString(CultureInfo.InvariantCulture); var fromY = from.Y.Value.ToString(CultureInfo.InvariantCulture); @@ -128,22 +86,14 @@ public static string GetWhere(string geometry_column, Point from, Point to, stri var where = ""; if (!hasZ) { - where = keepProjection ? - $"ST_Centroid(ST_Envelope({geometry_column})) && ST_MakeEnvelope({fromX}, {fromY}, {toX}, {toY}, {source_epsg}) {query}" : - $"ST_Centroid(ST_Envelope({geometry_column})) && st_transform(ST_MakeEnvelope({fromX}, {fromY}, {toX}, {toY}, 4326), {source_epsg}) {query}"; + where = $"ST_Centroid(ST_Envelope({geometry_column})) && ST_MakeEnvelope({fromX}, {fromY}, {toX}, {toY}, {source_epsg}) {query}"; } else { - var fromBox = keepProjection ? - $"st_setsrid(ST_MakePoint({fromX}, {fromY}, {from.Z.Value.ToString(CultureInfo.InvariantCulture)}), {source_epsg})" : - $"st_setsrid(ST_MakePoint({fromX}, {fromY}, {from.Z.Value.ToString(CultureInfo.InvariantCulture)}), 4979)"; - var toBox = keepProjection ? - $"st_setsrid(ST_MakePoint({toX}, {toY}, {to.Z.Value.ToString(CultureInfo.InvariantCulture)}), {source_epsg})" : - $"st_setsrid(ST_MakePoint({toX}, {toY}, {to.Z.Value.ToString(CultureInfo.InvariantCulture)}), 4979)"; + var fromBox = $"st_setsrid(ST_MakePoint({fromX}, {fromY}, {from.Z.Value.ToString(CultureInfo.InvariantCulture)}), {source_epsg})"; + var toBox = $"st_setsrid(ST_MakePoint({toX}, {toY}, {to.Z.Value.ToString(CultureInfo.InvariantCulture)}), {source_epsg})"; var geom = $"st_setsrid(st_makepoint((st_xmin({geometry_column}) + st_xmax({geometry_column}))/2,(st_ymin({geometry_column}) + st_ymax({geometry_column}))/2, (st_zmin({geometry_column}) + st_zmax({geometry_column}))/2), {source_epsg})"; - where = keepProjection ? - $"ST_3DIntersects({geom}, ST_3DMakeBox({fromBox}, {toBox})) {query}" : - $"ST_3DIntersects({geom}, st_transform(ST_3DMakeBox({fromBox}, {toBox}), {source_epsg})) {query}"; + where = $"ST_3DIntersects({geom}, ST_3DMakeBox({fromBox}, {toBox})) {query}"; } return where; diff --git a/src/b3dm.tileset/OctreeTiler.cs b/src/b3dm.tileset/OctreeTiler.cs index 1d97882a..c42ed92e 100644 --- a/src/b3dm.tileset/OctreeTiler.cs +++ b/src/b3dm.tileset/OctreeTiler.cs @@ -1,7 +1,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using B3dm.Tileset; using Npgsql; using subtree; using Wkx; @@ -43,7 +42,7 @@ public List GenerateTiles3D(BoundingBox3D bbox, int level, Tile3D tile, var where = inputTable.GetQueryClause(); - var numberOfFeatures = FeatureCountRepository.CountFeaturesInBox(conn, inputTable.TableName, inputTable.GeometryColumn, new Point(bbox.XMin, bbox.YMin, bbox.ZMin), new Point(bbox.XMax, bbox.YMax, bbox.ZMax), where, inputTable.EPSGCode, tilingSettings.KeepProjection, processedGeometries); + var numberOfFeatures = FeatureCountRepository.CountFeaturesInBox(conn, inputTable.TableName, inputTable.GeometryColumn, new Point(bbox.XMin, bbox.YMin, bbox.ZMin), new Point(bbox.XMax, bbox.YMax, bbox.ZMax), where, inputTable.EPSGCode, processedGeometries); if (numberOfFeatures == 0) { var t2 = new Tile3D(level, tile.X, tile.Y, tile.Z); t2.Available = false; @@ -76,10 +75,9 @@ public List GenerateTiles3D(BoundingBox3D bbox, int level, Tile3D tile, var bbox3d = new BoundingBox3D(xstart, ystart, z_start, xend, yend, zend); var bboxOctant = new BoundingBox(xstart, ystart, xend, yend); - var filteredProcessedGeometries = GeometryRepository.FilterHashesByEnvelope(conn, inputTable.TableName, inputTable.GeometryColumn, bboxOctant, inputTable.EPSGCode, localProcessedGeometries, tilingSettings.KeepProjection); var new_tile = new Tile3D(level, tile.X * 2 + x, tile.Y * 2 + y, tile.Z * 2 + z); - GenerateTiles3D(bbox3d, level, new_tile, tiles, tileBounds, filteredProcessedGeometries); + GenerateTiles3D(bbox3d, level, new_tile, tiles, tileBounds, localProcessedGeometries); } } } @@ -101,7 +99,7 @@ private HashSet CreateTileForLargestGeometries3D(BoundingBox3D bbox, int int target_srs = tilingSettings.KeepProjection ? inputTable.EPSGCode : 4978; var bbox1 = new double[] { bbox.XMin, bbox.YMin, bbox.XMax, bbox.YMax, bbox.ZMin, bbox.ZMax }; - var geometriesToProcess = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, bbox1, inputTable.EPSGCode, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, tilingSettings.KeepProjection, processedGeometries, tilingSettings.MaxFeaturesPerTile, tilingSettings.SortBy); + var geometriesToProcess = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, bbox1, inputTable.EPSGCode, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, processedGeometries, tilingSettings.MaxFeaturesPerTile, tilingSettings.SortBy); if (geometriesToProcess.Count > 0) { foreach (var geom in geometriesToProcess.Where(geom => !string.IsNullOrEmpty(geom.Hash))) { @@ -127,7 +125,7 @@ private void CreateTile3D(BoundingBox3D bbox, int level, Tile3D tile, List 0) { // Collect hashes of processed geometries diff --git a/src/b3dm.tileset/QuadtreeTiler.cs b/src/b3dm.tileset/QuadtreeTiler.cs index 45d70c98..d1ae083b 100644 --- a/src/b3dm.tileset/QuadtreeTiler.cs +++ b/src/b3dm.tileset/QuadtreeTiler.cs @@ -55,7 +55,7 @@ public List GenerateTiles(BoundingBox bbox, Tile tile, List tiles, i where += $" and {lodquery}"; } - var numberOfFeatures = FeatureCountRepository.CountFeaturesInBox(conn, inputTable.TableName, inputTable.GeometryColumn, new Point(bbox.XMin, bbox.YMin), new Point(bbox.XMax, bbox.YMax), where, source_epsg, keepProjection, processedGeometries); + var numberOfFeatures = FeatureCountRepository.CountFeaturesInBox(conn, inputTable.TableName, inputTable.GeometryColumn, new Point(bbox.XMin, bbox.YMin), new Point(bbox.XMax, bbox.YMax), where, source_epsg, processedGeometries); if (numberOfFeatures == 0) { tile.Available = false; @@ -82,9 +82,7 @@ public List GenerateTiles(BoundingBox bbox, Tile tile, List tiles, i var new_tile = new Tile(z, tile.X * 2 + x, tile.Y * 2 + y); new_tile.BoundingBox = bboxQuad.ToArray(); - var filteredProcessedGeometries = GeometryRepository.FilterHashesByEnvelope(conn, inputTable.TableName, inputTable.GeometryColumn, bboxQuad, source_epsg, localProcessedGeometries, keepProjection); - - GenerateTiles(bboxQuad, new_tile, tiles, lod, createGltf, keepProjection, filteredProcessedGeometries); + GenerateTiles(bboxQuad, new_tile, tiles, lod, createGltf, keepProjection, localProcessedGeometries); } } } @@ -107,7 +105,7 @@ private HashSet CreateTileForLargestGeometries(BoundingBox bbox, Tile ti int target_srs = keepProjection ? source_epsg : 4978; - var geometriesToProcess = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, tile.BoundingBox, source_epsg, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, keepProjection, processedGeometries, maxFeaturesPerTile, sortBy: sortBy); + var geometriesToProcess = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, tile.BoundingBox, source_epsg, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, processedGeometries, maxFeaturesPerTile, sortBy: sortBy); if (geometriesToProcess.Count > 0) { @@ -161,7 +159,7 @@ private void CreateTile(BoundingBox bbox, Tile tile, List tiles, string wh int target_srs = keepProjection ? source_epsg : 4978; - var geometries = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, tile.BoundingBox, source_epsg, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, keepProjection, processedGeometries, sortBy: sortBy); + var geometries = GeometryRepository.GetGeometrySubset(conn, inputTable.TableName, inputTable.GeometryColumn, tile.BoundingBox, source_epsg, target_srs, inputTable.ShadersColumn, inputTable.AttributeColumns, where, inputTable.RadiusColumn, processedGeometries, sortBy: sortBy); if (geometries.Count > 0) { // Collect hashes of processed geometries diff --git a/src/pg2b3dm.database.tests/UnitTest1.cs b/src/pg2b3dm.database.tests/UnitTest1.cs index 3a2ab4e7..a7a3b434 100644 --- a/src/pg2b3dm.database.tests/UnitTest1.cs +++ b/src/pg2b3dm.database.tests/UnitTest1.cs @@ -43,9 +43,12 @@ public void TestArvieuxBuildingsOctree() OutputDirectoryCreator.GetFolders("output"); var connectionString = _containerPostgres.GetConnectionString(); var conn = new NpgsqlConnection(connectionString); - var bbox_table = BoundingBoxRepository.GetBoundingBoxForTable(conn, "arvieux_batiments", "geom"); + // Get bbox in source projection (EPSG:5698) for spatial queries + var bbox_table = BoundingBoxRepository.GetBoundingBoxForTable(conn, "arvieux_batiments", "geom", true); + // Get WGS84 bbox separately for ECEF translation + var bbox_wgs84 = BoundingBoxRepository.GetBoundingBoxForTable(conn, "arvieux_batiments", "geom", false); - var center_wgs84 = bbox_table.bbox.GetCenter(); + var center_wgs84 = bbox_wgs84.bbox.GetCenter(); var translation = SpatialConverter.GeodeticToEcef((double)center_wgs84.X!, (double)center_wgs84.Y!, 0); var trans = new double[] { translation.X, translation.Y, }; diff --git a/src/pg2b3dm/Program.cs b/src/pg2b3dm/Program.cs index 817111a1..e2cc340b 100644 --- a/src/pg2b3dm/Program.cs +++ b/src/pg2b3dm/Program.cs @@ -124,12 +124,12 @@ static void Main(string[] args) Console.WriteLine($"Query bounding box of {inputTable.TableName}.{inputTable.GeometryColumn}..."); var where = (inputTable.Query != string.Empty ? $" where {inputTable.Query}" : String.Empty); - var bbox_table = BoundingBoxRepository.GetBoundingBoxForTable(conn, inputTable.TableName, inputTable.GeometryColumn, keepProjection, where); + var bbox_table = BoundingBoxRepository.GetBoundingBoxForTable(conn, inputTable.TableName, inputTable.GeometryColumn, true, where); var bbox = bbox_table.bbox; var zmin = bbox_table.zmin; var zmax = bbox_table.zmax; - var proj = keepProjection ? $"EPSG:{source_epsg}" : $"EPSG:4326 (WGS84)"; + var proj = $"EPSG:{source_epsg}"; Console.WriteLine($"Bounding box for {inputTable.TableName}.{inputTable.GeometryColumn} ({proj}): " + $"{Math.Round(bbox.XMin, 8)}, {Math.Round(bbox.YMin, 8)}, " + $"{Math.Round(bbox.XMax, 8)}, {Math.Round(bbox.YMax, 8)}"); @@ -148,14 +148,23 @@ static void Main(string[] args) Console.WriteLine($"Attribute columns: {att}"); var center = bbox.GetCenter(); - Console.WriteLine($"Center ({proj}): {center.X}, {center.Y}"); + var center_z = (zmin + zmax) / 2; + Console.WriteLine($"Center ({proj}): {center.X}, {center.Y}, {center_z}"); Tiles3DExtensions.RegisterExtensions(); - var translation = keepProjection ? - [(double)center.X, (double)center.Y, 0] : - Translation.ToEcef(center); - Console.WriteLine($"Translation: {String.Join(',', translation)}"); + double[] translation = [(double)center.X, (double)center.Y, center_z]; + + var bbox_wgs84 = bbox; // fallback, only set when !keepProjection + if(!keepProjection) { + // Convert bbox to EPSG:4979 for ECEF translation and bounding volume region + var bbox_4979 = BoundingBoxRepository.GetBoundingBoxAs4979(conn, bbox_table, source_epsg); + bbox_wgs84 = bbox_4979.bbox; + var center_wgs84 = bbox_wgs84.GetCenter(); + var p = new Wkx.Point((double)center_wgs84.X, (double)center_wgs84.Y, center_z); + translation = Translation.ToEcef(p); + } + Console.WriteLine($"Translation (EPSG:4978): {String.Join(',', translation)}"); var lodcolumn = o.LodColumn; var addOutlines = (bool)o.AddOutlines; @@ -197,7 +206,7 @@ static void Main(string[] args) var rootBoundingVolumeRegion = keepProjection ? bbox.ToRegion(zmin, zmax) : - bbox.ToRadians().ToRegion(zmin, zmax); + bbox_wgs84.ToRadians().ToRegion(zmin, zmax); Console.WriteLine($"Maximum features per tile: " + maxFeaturesPerTile);