From b07c2f3e7ccaf4de563ca87bc9f9e8c9d3dbbce0 Mon Sep 17 00:00:00 2001 From: Collin Beczak Date: Thu, 12 Feb 2026 13:08:52 -0600 Subject: [PATCH 1/6] Implement tile aggregation system for efficient map display - Introduced a new `TileAggregate` model to represent pre-computed tile data, including task counts and centroid coordinates. - Added `TileAggregateRepository` for database interactions, enabling retrieval and management of tile aggregates. - Developed `TileAggregateService` to handle tile data processing, including filtering by difficulty and global status. - Enhanced `TaskController` with a new endpoint to fetch task tiles, returning either clusters or individual task markers based on task count. - Implemented a scheduled job for refreshing tile aggregates, ensuring up-to-date data for map displays. - Updated configuration to include tile refresh interval and batch size settings. - Created SQL migrations for the new tile aggregation tables and functions, supporting efficient data management. --- app/org/maproulette/Config.scala | 1 + .../framework/controller/TaskController.scala | 65 ++ .../framework/model/TileAggregate.scala | 108 ++++ .../repository/TileAggregateRepository.scala | 567 +++++++++++++++++ .../framework/service/NominatimService.scala | 51 +- .../framework/service/ServiceManager.scala | 5 +- .../service/TileAggregateService.scala | 586 ++++++++++++++++++ app/org/maproulette/jobs/Scheduler.scala | 7 + app/org/maproulette/jobs/SchedulerActor.scala | 21 + conf/application.conf | 3 + conf/evolutions/default/107.sql | 84 +++ conf/v2_route/task.api | 64 ++ test/org/maproulette/utils/TestSpec.scala | 4 +- 13 files changed, 1549 insertions(+), 17 deletions(-) create mode 100644 app/org/maproulette/framework/model/TileAggregate.scala create mode 100644 app/org/maproulette/framework/repository/TileAggregateRepository.scala create mode 100644 app/org/maproulette/framework/service/TileAggregateService.scala create mode 100644 conf/evolutions/default/107.sql diff --git a/app/org/maproulette/Config.scala b/app/org/maproulette/Config.scala index 47fb8d769..ca441bc1d 100644 --- a/app/org/maproulette/Config.scala +++ b/app/org/maproulette/Config.scala @@ -348,6 +348,7 @@ object Config { val KEY_SCHEDULER_SNAPSHOT_CHALLENGES_INTERVAL = s"$SUB_GROUP_SCHEDULER.challengesSnapshot.interval" val KEY_SCHEDULER_SNAPSHOT_CHALLENGES_START = s"$SUB_GROUP_SCHEDULER.challengesSnapshot.startTime" + val KEY_SCHEDULER_TILE_REFRESH_INTERVAL = s"$SUB_GROUP_SCHEDULER.tileRefresh.interval" val KEY_MAPROULETTE_FRONTEND = s"$GROUP_MAPROULETTE.frontend" val SUB_GROUP_MAPILLARY = s"$GROUP_MAPROULETTE.mapillary" diff --git a/app/org/maproulette/framework/controller/TaskController.scala b/app/org/maproulette/framework/controller/TaskController.scala index d3f2397fb..cdaefd8f8 100644 --- a/app/org/maproulette/framework/controller/TaskController.scala +++ b/app/org/maproulette/framework/controller/TaskController.scala @@ -327,6 +327,71 @@ class TaskController @Inject() ( } } + /** + * Gets task data using pre-computed tile aggregates for efficient map display at scale. + * Uses a tile pyramid system with pre-computed counts broken down by difficulty × global. + * + * Behavior by filter: + * - difficulty & global: Filtered from pre-computed tile data (fast) + * - location_id: Recursive tile drilling until within polygon or < 2000 tasks + * - keywords: Falls back to dynamic query (challenge-level filter, not pre-computed) + * + * All fetched data is re-clustered into ~80 clusters for display. + * When total tasks < 2000, returns individual task markers instead of clusters. + * + * @param z Zoom level (0-14 for pre-computed tiles) + * @param bounds Comma-separated bounding box: left,bottom,right,top + * @param global Whether to include global challenges + * @param location_id Optional Nominatim place_id for polygon filtering + * @param keywords Optional keywords filter (triggers fallback to dynamic query) + * @param difficulty Optional difficulty filter (1=Easy, 2=Normal, 3=Expert) + * @return TaskMarkerResponse with totalCount and either clusters or tasks (with overlaps) + */ + def getTaskTiles( + z: Int, + bounds: String, + global: Boolean, + location_id: Option[Long], + keywords: Option[String], + difficulty: Option[Int] + ): Action[AnyContent] = Action.async { implicit request => + this.sessionManager.userAwareRequest { implicit user => + // Clamp zoom to valid range + val validZoom = math.max(0, math.min(20, z)) + + // Only allow valid difficulty values (1, 2, 3) + val validDifficulty = difficulty.filter(d => d >= 1 && d <= 3) + + // Parse and validate bounding box + val boundingBox = try { + bounds.split(",").map(_.trim.toDouble).toList match { + case List(left, bottom, right, top) => + // Clamp coordinates to valid ranges + val clampedLeft = math.max(-180.0, math.min(180.0, left)) + val clampedRight = math.max(-180.0, math.min(180.0, right)) + val clampedBottom = math.max(-90.0, math.min(90.0, bottom)) + val clampedTop = math.max(-90.0, math.min(90.0, top)) + SearchLocation(clampedLeft, clampedBottom, clampedRight, clampedTop) + case _ => + SearchLocation(-180.0, -90.0, 180.0, 90.0) + } + } catch { + case _: NumberFormatException => + SearchLocation(-180.0, -90.0, 180.0, 90.0) + } + + val response = this.serviceManager.tileAggregate.getTileData( + validZoom, + boundingBox, + validDifficulty, + global, + location_id, + keywords + ) + Ok(Json.toJson(response)) + } + } + // for getting more detailed task marker data on individul makrers // def getTaskMarkerData(id: Long): Action[AnyContent] = Action.async { implicit request => // this.sessionManager.userAwareRequest { implicit user => diff --git a/app/org/maproulette/framework/model/TileAggregate.scala b/app/org/maproulette/framework/model/TileAggregate.scala new file mode 100644 index 000000000..97ccbf1f5 --- /dev/null +++ b/app/org/maproulette/framework/model/TileAggregate.scala @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2020 MapRoulette contributors (see CONTRIBUTORS.md). + * Licensed under the Apache License, Version 2.0 (see LICENSE). + */ +package org.maproulette.framework.model +import play.api.libs.json._ + +case class ClusterPoint( + lat: Double, + lng: Double, + count: Int +) + +object ClusterPoint { + implicit val clusterPointFormat: Format[ClusterPoint] = Json.format[ClusterPoint] +} + +case class FilterCounts( + d1_gf: Int = 0, + d1_gt: Int = 0, + d2_gf: Int = 0, + d2_gt: Int = 0, + d3_gf: Int = 0, + d3_gt: Int = 0, + d0_gf: Int = 0, + d0_gt: Int = 0 +) { + + /** + * Get count for specific difficulty and global filters + * + * @param difficulty Optional difficulty filter (1, 2, 3) + * @param global Whether to include global challenges (true = all, false = non-global only) + * @return Filtered count + */ + def getFilteredCount(difficulty: Option[Int], global: Boolean): Int = { + difficulty match { + case Some(1) => if (global) d1_gf + d1_gt else d1_gf + case Some(2) => if (global) d2_gf + d2_gt else d2_gf + case Some(3) => if (global) d3_gf + d3_gt else d3_gf + case Some(_) => + // Unknown difficulty values use d0 (unset/other) bucket + if (global) d0_gf + d0_gt else d0_gf + case None => + // No filter: sum all difficulty levels + if (global) d1_gf + d1_gt + d2_gf + d2_gt + d3_gf + d3_gt + d0_gf + d0_gt + else d1_gf + d2_gf + d3_gf + d0_gf + } + } + + /** + * Get total count (all combinations) + */ + def total: Int = d1_gf + d1_gt + d2_gf + d2_gt + d3_gf + d3_gt + d0_gf + d0_gt +} + +object FilterCounts { + implicit val filterCountsFormat: Format[FilterCounts] = Json.format[FilterCounts] + + def fromJson(json: JsValue): FilterCounts = { + FilterCounts( + d1_gf = (json \ "d1_gf").asOpt[Int].getOrElse(0), + d1_gt = (json \ "d1_gt").asOpt[Int].getOrElse(0), + d2_gf = (json \ "d2_gf").asOpt[Int].getOrElse(0), + d2_gt = (json \ "d2_gt").asOpt[Int].getOrElse(0), + d3_gf = (json \ "d3_gf").asOpt[Int].getOrElse(0), + d3_gt = (json \ "d3_gt").asOpt[Int].getOrElse(0), + d0_gf = (json \ "d0_gf").asOpt[Int].getOrElse(0), + d0_gt = (json \ "d0_gt").asOpt[Int].getOrElse(0) + ) + } +} + +/** + * Represents a pre-computed tile aggregate for efficient map display at scale. + * Uses Web Mercator tile coordinates (z/x/y). + * Only tracks tasks with status 0, 3, or 6. + * + * @param z Zoom level (0-14 for pre-computed) + * @param x Tile X coordinate + * @param y Tile Y coordinate + * @param taskCount Total tasks in this tile (all filters) + * @param countsByFilter Counts broken down by difficulty × global + * @param centroidLat Centroid latitude of all tasks in tile + * @param centroidLng Centroid longitude of all tasks in tile + */ +case class TileAggregate( + z: Int, + x: Int, + y: Int, + taskCount: Int, + countsByFilter: FilterCounts, + centroidLat: Double, + centroidLng: Double +) { + + /** + * Get the filtered count for this tile based on difficulty and global filters + */ + def getFilteredCount(difficulty: Option[Int], global: Boolean): Int = { + countsByFilter.getFilteredCount(difficulty, global) + } +} + +object TileAggregate { + implicit val tileAggregateWrites: Writes[TileAggregate] = Json.writes[TileAggregate] + implicit val tileAggregateReads: Reads[TileAggregate] = Json.reads[TileAggregate] +} diff --git a/app/org/maproulette/framework/repository/TileAggregateRepository.scala b/app/org/maproulette/framework/repository/TileAggregateRepository.scala new file mode 100644 index 000000000..342f20e91 --- /dev/null +++ b/app/org/maproulette/framework/repository/TileAggregateRepository.scala @@ -0,0 +1,567 @@ +/* + * Copyright (C) 2020 MapRoulette contributors (see CONTRIBUTORS.md). + * Licensed under the Apache License, Version 2.0 (see LICENSE). + */ + +package org.maproulette.framework.repository + +import java.sql.Connection + +import anorm._ +import anorm.SqlParser.{get, int, double} +import javax.inject.{Inject, Singleton} +import org.maproulette.framework.model.{TileAggregate, FilterCounts, TaskMarker, TaskMarkerLocation, ClusterPoint} +import org.maproulette.session.SearchLocation +import play.api.db.Database +import play.api.libs.json.Json + +/** + * Repository for accessing pre-computed tile aggregates. + * Tiles track tasks with status 0, 3, or 6, with breakdowns by difficulty and global. + */ +@Singleton +class TileAggregateRepository @Inject() (override val db: Database) extends RepositoryMixin { + implicit val baseTable: String = "tile_aggregates" + + /** + * Parser for converting database rows to TileAggregate objects + */ + private val tileAggregateParser: RowParser[TileAggregate] = { + get[Int]("z") ~ + get[Int]("x") ~ + get[Int]("y") ~ + get[Int]("task_count") ~ + get[Option[String]]("counts_by_filter") ~ + get[Option[Double]]("centroid_lat") ~ + get[Option[Double]]("centroid_lng") map { + case z ~ x ~ y ~ taskCount ~ countsJson ~ centroidLat ~ centroidLng => + val filterCounts = countsJson + .map { json => + try { + FilterCounts.fromJson(Json.parse(json)) + } catch { + case _: Exception => FilterCounts() + } + } + .getOrElse(FilterCounts()) + + TileAggregate( + z, + x, + y, + taskCount, + filterCounts, + centroidLat.getOrElse(0.0), + centroidLng.getOrElse(0.0) + ) + } + } + + /** + * Get tiles for a bounding box at a specific zoom level. + * Handles anti-meridian crossing (when bounds.left > bounds.right). + */ + def getTilesInBounds( + zoom: Int, + bounds: SearchLocation + )(implicit c: Option[Connection] = None): List[TileAggregate] = { + this.withMRConnection { implicit c => + val minY = latToTileY(bounds.top, zoom) + val maxY = latToTileY(bounds.bottom, zoom) + + + if (bounds.left > bounds.right) { + + val leftMinX = lngToTileX(bounds.left, zoom) + val leftMaxX = (1 << zoom) - 1 + val rightMinX = 0 + val rightMaxX = lngToTileX(bounds.right, zoom) + + val leftTiles = SQL""" + SELECT z, x, y, task_count, counts_by_filter::text as counts_by_filter, + centroid_lat, centroid_lng + FROM tile_aggregates + WHERE z = $zoom + AND x >= $leftMinX AND x <= $leftMaxX + AND y >= $minY AND y <= $maxY + AND task_count > 0 + """.as(tileAggregateParser.*) + + val rightTiles = SQL""" + SELECT z, x, y, task_count, counts_by_filter::text as counts_by_filter, + centroid_lat, centroid_lng + FROM tile_aggregates + WHERE z = $zoom + AND x >= $rightMinX AND x <= $rightMaxX + AND y >= $minY AND y <= $maxY + AND task_count > 0 + """.as(tileAggregateParser.*) + + leftTiles ++ rightTiles + } else { + + val minX = lngToTileX(bounds.left, zoom) + val maxX = lngToTileX(bounds.right, zoom) + + SQL""" + SELECT z, x, y, task_count, counts_by_filter::text as counts_by_filter, + centroid_lat, centroid_lng + FROM tile_aggregates + WHERE z = $zoom + AND x >= $minX AND x <= $maxX + AND y >= $minY AND y <= $maxY + AND task_count > 0 + """.as(tileAggregateParser.*) + } + } + } + + /** + * Get a single tile by coordinates + */ + def getTile(z: Int, x: Int, y: Int)( + implicit c: Option[Connection] = None + ): Option[TileAggregate] = { + this.withMRConnection { implicit c => + SQL""" + SELECT z, x, y, task_count, counts_by_filter::text as counts_by_filter, + centroid_lat, centroid_lng + FROM tile_aggregates + WHERE z = $z AND x = $x AND y = $y + """.as(tileAggregateParser.singleOpt) + } + } + + /** + * Get child tiles for recursive drilling (zoom level z+1) + */ + def getChildTiles(z: Int, x: Int, y: Int)( + implicit c: Option[Connection] = None + ): List[TileAggregate] = { + this.withMRConnection { implicit c => + + val childZ = z + 1 + val childXMin = x * 2 + val childXMax = x * 2 + 1 + val childYMin = y * 2 + val childYMax = y * 2 + 1 + + SQL""" + SELECT z, x, y, task_count, counts_by_filter::text as counts_by_filter, + centroid_lat, centroid_lng + FROM tile_aggregates + WHERE z = $childZ + AND x >= $childXMin AND x <= $childXMax + AND y >= $childYMin AND y <= $childYMax + AND task_count > 0 + """.as(tileAggregateParser.*) + } + } + + /** + * Get tiles at a specific zoom level whose centroids fall within a polygon. + * First filters by bounding box (fast indexed lookup), then by polygon containment. + * The polygon is simplified to reduce complexity for faster spatial queries. + */ + def getTilesInPolygon( + zoom: Int, + polygonWkt: String, + bounds: SearchLocation + )(implicit c: Option[Connection] = None): List[TileAggregate] = { + this.withMRConnection { implicit c => + val minX = lngToTileX(bounds.left, zoom) + val maxX = lngToTileX(bounds.right, zoom) + val minY = latToTileY(bounds.top, zoom) + val maxY = latToTileY(bounds.bottom, zoom) + + SQL""" + WITH simplified AS ( + SELECT ST_Simplify(ST_GeomFromText($polygonWkt, 4326), $SIMPLIFY_TOLERANCE) as geom + ) + SELECT z, x, y, task_count, counts_by_filter::text as counts_by_filter, + centroid_lat, centroid_lng + FROM tile_aggregates + CROSS JOIN simplified + WHERE z = $zoom + AND x >= $minX AND x <= $maxX + AND y >= $minY AND y <= $maxY + AND task_count > 0 + AND ST_Contains(simplified.geom, ST_SetSRID(ST_MakePoint(centroid_lng, centroid_lat), 4326)) + """.as(tileAggregateParser.*) + } + } + + /** + * Get tile bounds as (west, south, east, north) + */ + def getTileBounds(z: Int, x: Int, y: Int): (Double, Double, Double, Double) = { + val west = tileToLng(x, z) + val east = tileToLng(x + 1, z) + val north = tileToLat(y, z) + val south = tileToLat(y + 1, z) + (west, south, east, north) + } + + /** + * Get tile bounds as SearchLocation + */ + def getTileBoundsAsSearchLocation(z: Int, x: Int, y: Int): SearchLocation = { + val (west, south, east, north) = getTileBounds(z, x, y) + SearchLocation(west, south, east, north) + } + + /** + * Fetch all task markers in a bounding box with a single query. + * Much more efficient than querying per-tile when total count is low. + */ + def getTaskMarkersInBounds( + bounds: SearchLocation, + difficulty: Option[Int] = None, + global: Boolean = false, + limit: Int = 2000 + )(implicit c: Option[Connection] = None): List[TaskMarker] = { + this.withMRConnection { implicit c => + val left = bounds.left + val bottom = bounds.bottom + val right = bounds.right + val top = bounds.top + + // SQL SAFETY: These use #$ string interpolation which is safe here because: + // - globalFilter is a hardcoded string literal (no user input) + // - difficultyFilter uses a validated Int from difficulty.map, not user-provided strings + val globalFilter = if (!global) "AND c.is_global = false" else "" + val difficultyFilter = difficulty.map(d => s"AND c.difficulty = $d").getOrElse("") + + SQL""" + SELECT DISTINCT tasks.id, ST_Y(tasks.location) as lat, ST_X(tasks.location) as lng, + tasks.status, tasks.priority, tasks.bundle_id, l.user_id as locked_by + FROM tasks + INNER JOIN challenges c ON c.id = tasks.parent_id + INNER JOIN projects p ON p.id = c.parent_id + LEFT JOIN locked l ON l.item_id = tasks.id AND l.item_type = 2 + WHERE tasks.location && ST_MakeEnvelope($left, $bottom, $right, $top, 4326) + AND ST_Intersects(tasks.location, ST_MakeEnvelope($left, $bottom, $right, $top, 4326)) + AND tasks.status IN (0, 3, 6) + AND c.deleted = false AND c.enabled = true AND c.is_archived = false + AND p.deleted = false AND p.enabled = true + #$globalFilter + #$difficultyFilter + LIMIT $limit + """.as(taskMarkerParser.*) + } + } + + /** + * Count tasks within a polygon (for deciding fetch vs cluster strategy) + */ + def countTasksInPolygon( + polygonWkt: String, + difficulty: Option[Int] = None, + global: Boolean = false + )(implicit c: Option[Connection] = None): Int = { + this.withMRConnection { implicit c => + val globalFilter = if (!global) "AND c.is_global = false" else "" + val difficultyFilter = difficulty.map(d => s"AND c.difficulty = $d").getOrElse("") + + SQL""" + SELECT COUNT(DISTINCT tasks.id)::int as count + FROM tasks + INNER JOIN challenges c ON c.id = tasks.parent_id + INNER JOIN projects p ON p.id = c.parent_id + WHERE tasks.location && ST_GeomFromText($polygonWkt, 4326) + AND ST_Intersects(tasks.location, ST_GeomFromText($polygonWkt, 4326)) + AND tasks.status IN (0, 3, 6) + AND c.deleted = false AND c.enabled = true AND c.is_archived = false + AND p.deleted = false AND p.enabled = true + #$globalFilter + #$difficultyFilter + """.as(SqlParser.int("count").single) + } + } + + /** + * Fetch task markers within a polygon (for location_id filtering) + * Uses bounding box operator for index acceleration. + */ + def getTaskMarkersInPolygon( + polygonWkt: String, + difficulty: Option[Int] = None, + global: Boolean = false, + limit: Option[Int] = None + )(implicit c: Option[Connection] = None): List[TaskMarker] = { + this.withMRConnection { implicit c => + val globalFilter = if (!global) "AND c.is_global = false" else "" + val difficultyFilter = difficulty.map(d => s"AND c.difficulty = $d").getOrElse("") + val limitClause = limit.map(l => s"LIMIT $l").getOrElse("") + + SQL""" + SELECT DISTINCT tasks.id, ST_Y(tasks.location) as lat, ST_X(tasks.location) as lng, + tasks.status, tasks.priority, tasks.bundle_id, l.user_id as locked_by + FROM tasks + INNER JOIN challenges c ON c.id = tasks.parent_id + INNER JOIN projects p ON p.id = c.parent_id + LEFT JOIN locked l ON l.item_id = tasks.id AND l.item_type = 2 + WHERE tasks.location && ST_GeomFromText($polygonWkt, 4326) + AND ST_Intersects(tasks.location, ST_GeomFromText($polygonWkt, 4326)) + AND tasks.status IN (0, 3, 6) + AND c.deleted = false AND c.enabled = true AND c.is_archived = false + AND p.deleted = false AND p.enabled = true + #$globalFilter + #$difficultyFilter + #$limitClause + """.as(taskMarkerParser.*) + } + } + + + private val MAX_CLUSTER_POINTS = 50000 + + /** + * Get clustered task markers within a polygon using PostGIS kmeans. + * Returns cluster centroids with counts. + * For very large datasets (>50K points), uses statistical sampling to maintain performance. + */ + def getClusteredTasksInPolygon( + polygonWkt: String, + difficulty: Option[Int] = None, + global: Boolean = false, + numClusters: Int = 80 + )(implicit c: Option[Connection] = None): List[ClusterPoint] = { + this.withMRConnection { implicit c => + val globalFilter = if (!global) "AND c.is_global = false" else "" + val difficultyFilter = difficulty.map(d => s"AND c.difficulty = $d").getOrElse("") + + // Use sampling for very large datasets to prevent slow K-means clustering + // The sample maintains spatial distribution while limiting compute time + SQL""" + WITH task_points AS ( + SELECT tasks.location + FROM tasks + INNER JOIN challenges c ON c.id = tasks.parent_id + INNER JOIN projects p ON p.id = c.parent_id + WHERE tasks.location && ST_GeomFromText($polygonWkt, 4326) + AND ST_Intersects(tasks.location, ST_GeomFromText($polygonWkt, 4326)) + AND tasks.status IN (0, 3, 6) + AND c.deleted = false AND c.enabled = true AND c.is_archived = false + AND p.deleted = false AND p.enabled = true + #$globalFilter + #$difficultyFilter + ), + point_count AS ( + SELECT COUNT(*) as total FROM task_points + ), + sampled_points AS ( + SELECT location + FROM task_points + WHERE (SELECT total FROM point_count) <= $MAX_CLUSTER_POINTS + OR random() < ($MAX_CLUSTER_POINTS::float / GREATEST((SELECT total FROM point_count), 1)) + ), + clustered AS ( + SELECT ST_ClusterKMeans(location, $numClusters) OVER() as cluster_id, location + FROM sampled_points + ) + SELECT + AVG(ST_Y(location)) as lat, + AVG(ST_X(location)) as lng, + COUNT(*)::int as count + FROM clustered + GROUP BY cluster_id + HAVING COUNT(*) > 0 + """.as(clusterPointParser.*) + } + } + + private val clusterPointParser: RowParser[ClusterPoint] = { + get[Double]("lat") ~ + get[Double]("lng") ~ + get[Int]("count") map { + case lat ~ lng ~ count => + ClusterPoint(lat, lng, count) + } + } + + // Simplification tolerance in degrees (~1km at equator) - reduces polygon complexity significantly + private val SIMPLIFY_TOLERANCE = 0.01 + + /** + * Count tasks within a simplified polygon. + * Uses ST_Simplify to reduce polygon complexity for faster queries. + */ + def countTasksInPolygonSimplified( + polygonWkt: String, + difficulty: Option[Int] = None, + global: Boolean = false + )(implicit c: Option[Connection] = None): Int = { + this.withMRConnection { implicit c => + val globalFilter = if (!global) "AND c.is_global = false" else "" + val difficultyFilter = difficulty.map(d => s"AND c.difficulty = $d").getOrElse("") + + SQL""" + WITH simplified AS ( + SELECT ST_Simplify(ST_GeomFromText($polygonWkt, 4326), $SIMPLIFY_TOLERANCE) as geom + ) + SELECT COUNT(DISTINCT tasks.id)::int as count + FROM tasks + CROSS JOIN simplified + INNER JOIN challenges c ON c.id = tasks.parent_id + INNER JOIN projects p ON p.id = c.parent_id + WHERE tasks.location && simplified.geom + AND ST_Intersects(tasks.location, simplified.geom) + AND tasks.status IN (0, 3, 6) + AND c.deleted = false AND c.enabled = true AND c.is_archived = false + AND p.deleted = false AND p.enabled = true + #$globalFilter + #$difficultyFilter + """.as(SqlParser.int("count").single) + } + } + + /** + * Fetch task markers within a simplified polygon. + */ + def getTaskMarkersInPolygonSimplified( + polygonWkt: String, + difficulty: Option[Int] = None, + global: Boolean = false, + limit: Option[Int] = None + )(implicit c: Option[Connection] = None): List[TaskMarker] = { + this.withMRConnection { implicit c => + val globalFilter = if (!global) "AND c.is_global = false" else "" + val difficultyFilter = difficulty.map(d => s"AND c.difficulty = $d").getOrElse("") + val limitClause = limit.map(l => s"LIMIT $l").getOrElse("") + + SQL""" + WITH simplified AS ( + SELECT ST_Simplify(ST_GeomFromText($polygonWkt, 4326), $SIMPLIFY_TOLERANCE) as geom + ) + SELECT DISTINCT tasks.id, ST_Y(tasks.location) as lat, ST_X(tasks.location) as lng, + tasks.status, tasks.priority, tasks.bundle_id, l.user_id as locked_by + FROM tasks + CROSS JOIN simplified + INNER JOIN challenges c ON c.id = tasks.parent_id + INNER JOIN projects p ON p.id = c.parent_id + LEFT JOIN locked l ON l.item_id = tasks.id AND l.item_type = 2 + WHERE tasks.location && simplified.geom + AND ST_Intersects(tasks.location, simplified.geom) + AND tasks.status IN (0, 3, 6) + AND c.deleted = false AND c.enabled = true AND c.is_archived = false + AND p.deleted = false AND p.enabled = true + #$globalFilter + #$difficultyFilter + #$limitClause + """.as(taskMarkerParser.*) + } + } + + /** + * Get clustered task markers within a simplified polygon. + */ + def getClusteredTasksInPolygonSimplified( + polygonWkt: String, + difficulty: Option[Int] = None, + global: Boolean = false, + numClusters: Int = 80 + )(implicit c: Option[Connection] = None): List[ClusterPoint] = { + this.withMRConnection { implicit c => + val globalFilter = if (!global) "AND c.is_global = false" else "" + val difficultyFilter = difficulty.map(d => s"AND c.difficulty = $d").getOrElse("") + + SQL""" + WITH simplified AS ( + SELECT ST_Simplify(ST_GeomFromText($polygonWkt, 4326), $SIMPLIFY_TOLERANCE) as geom + ), + task_points AS ( + SELECT tasks.location + FROM tasks + CROSS JOIN simplified + INNER JOIN challenges c ON c.id = tasks.parent_id + INNER JOIN projects p ON p.id = c.parent_id + WHERE tasks.location && simplified.geom + AND ST_Intersects(tasks.location, simplified.geom) + AND tasks.status IN (0, 3, 6) + AND c.deleted = false AND c.enabled = true AND c.is_archived = false + AND p.deleted = false AND p.enabled = true + #$globalFilter + #$difficultyFilter + ), + point_count AS ( + SELECT COUNT(*) as total FROM task_points + ), + sampled_points AS ( + SELECT location + FROM task_points + WHERE (SELECT total FROM point_count) <= $MAX_CLUSTER_POINTS + OR random() < ($MAX_CLUSTER_POINTS::float / GREATEST((SELECT total FROM point_count), 1)) + ), + clustered AS ( + SELECT ST_ClusterKMeans(location, $numClusters) OVER() as cluster_id, location + FROM sampled_points + ) + SELECT + AVG(ST_Y(location)) as lat, + AVG(ST_X(location)) as lng, + COUNT(*)::int as count + FROM clustered + GROUP BY cluster_id + HAVING COUNT(*) > 0 + """.as(clusterPointParser.*) + } + } + + private val taskMarkerParser: RowParser[TaskMarker] = { + get[Long]("id") ~ + get[Double]("lat") ~ + get[Double]("lng") ~ + get[Int]("status") ~ + get[Int]("priority") ~ + get[Option[Long]]("bundle_id") ~ + get[Option[Long]]("locked_by") map { + case id ~ lat ~ lng ~ status ~ priority ~ bundleId ~ lockedBy => + TaskMarker(id, TaskMarkerLocation(lat, lng), status, priority, bundleId, lockedBy) + } + } + + /** + * Rebuild all tiles for a specific zoom level + */ + def rebuildZoomLevel(zoom: Int)(implicit c: Option[Connection] = None): Int = { + this.withMRTransaction { implicit c => + SQL"SELECT rebuild_zoom_level($zoom)" + .as(SqlParser.int("rebuild_zoom_level").single) + } + } + + /** + * Get total count of pre-computed tiles + */ + def getTotalTileCount()(implicit c: Option[Connection] = None): Int = { + this.withMRConnection { implicit c => + SQL"SELECT COUNT(*)::int as count FROM tile_aggregates" + .as(SqlParser.int("count").single) + } + } + + // Web Mercator coordinate conversion functions + def lngToTileX(lng: Double, zoom: Int): Int = { + math.floor((lng + 180.0) / 360.0 * (1 << zoom)).toInt + } + + def latToTileY(lat: Double, zoom: Int): Int = { + val latClamped = math.max(-85.0511, math.min(85.0511, lat)) + val latRad = math.toRadians(latClamped) + math + .floor( + (1.0 - math.log(math.tan(latRad) + 1.0 / math.cos(latRad)) / math.Pi) / 2.0 * (1 << zoom) + ) + .toInt + } + + def tileToLng(x: Int, zoom: Int): Double = { + x.toDouble / (1 << zoom) * 360.0 - 180.0 + } + + def tileToLat(y: Int, zoom: Int): Double = { + val n = math.Pi - 2.0 * math.Pi * y.toDouble / (1 << zoom) + math.toDegrees(math.atan(math.sinh(n))) + } +} diff --git a/app/org/maproulette/framework/service/NominatimService.scala b/app/org/maproulette/framework/service/NominatimService.scala index 8c9aac969..227746f61 100644 --- a/app/org/maproulette/framework/service/NominatimService.scala +++ b/app/org/maproulette/framework/service/NominatimService.scala @@ -47,6 +47,7 @@ class NominatimService @Inject() (wsClient: WSClient)(implicit ec: ExecutionCont /** * Fetches polygon geometry from Nominatim API (not cached) + * Note: Uses blocking call - consider using async version for high-throughput scenarios. * * @param placeId The Nominatim place_id * @return Option containing the WKT polygon string @@ -57,6 +58,7 @@ class NominatimService @Inject() (wsClient: WSClient)(implicit ec: ExecutionCont val futureResponse = wsClient .url(url) + .withRequestTimeout(REQUEST_TIMEOUT) .addQueryStringParameters( "place_id" -> placeId.toString, "format" -> "json", @@ -68,7 +70,7 @@ class NominatimService @Inject() (wsClient: WSClient)(implicit ec: ExecutionCont ) .get() - val response = Await.result(futureResponse, REQUEST_TIMEOUT) + val response = Await.result(futureResponse, REQUEST_TIMEOUT + 1.second) if (response.status == 200) { val json = response.json @@ -79,35 +81,54 @@ class NominatimService @Inject() (wsClient: WSClient)(implicit ec: ExecutionCont // Convert the GeoJSON geometry to a WKT string for PostGIS convertGeoJSONToWKT(geometry) case None => + None } } else { + None } } catch { - case e: Exception => + case _: java.util.concurrent.TimeoutException => + + None + case _: Exception => + None } } /** - * Converts a GeoJSON geometry object to WKT (Well-Known Text) format for PostGIS + * Converts a GeoJSON geometry object to WKT (Well-Known Text) format for PostGIS. + * Handles Polygon, MultiPolygon, Point, and LineString geometry types. * * @param geometry The GeoJSON geometry object - * @return Option containing the WKT string, or None if conversion fails + * @return Option containing the WKT string, or None if conversion fails or type unsupported */ private def convertGeoJSONToWKT(geometry: JsObject): Option[String] = { - val geometryType = (geometry \ "type").asOpt[String] - val coordinates = (geometry \ "coordinates").asOpt[JsArray] - - (geometryType, coordinates) match { - case (Some("Polygon"), Some(coords)) => - Some(polygonToWKT(coords)) - case (Some("MultiPolygon"), Some(coords)) => - Some(multiPolygonToWKT(coords)) - case (Some("Point"), Some(coords)) => - Some(pointToWKT(coords)) - case _ => + try { + val geometryType = (geometry \ "type").asOpt[String] + val coordinates = (geometry \ "coordinates").asOpt[JsArray] + + (geometryType, coordinates) match { + case (Some("Polygon"), Some(coords)) if coords.value.nonEmpty => + Some(polygonToWKT(coords)) + case (Some("MultiPolygon"), Some(coords)) if coords.value.nonEmpty => + Some(multiPolygonToWKT(coords)) + case (Some("Point"), Some(coords)) if coords.value.size >= 2 => + Some(pointToWKT(coords)) + case (Some("LineString"), Some(coords)) if coords.value.size >= 2 => + + + None + case (Some("GeometryCollection"), _) => + + None + case _ => + None + } + } catch { + case _: Exception => None } } diff --git a/app/org/maproulette/framework/service/ServiceManager.scala b/app/org/maproulette/framework/service/ServiceManager.scala index 55ad2fab8..291fc7b35 100644 --- a/app/org/maproulette/framework/service/ServiceManager.scala +++ b/app/org/maproulette/framework/service/ServiceManager.scala @@ -39,7 +39,8 @@ class ServiceManager @Inject() ( notificationService: Provider[NotificationService], leaderboardService: Provider[LeaderboardService], taskHistoryService: Provider[TaskHistoryService], - nominatimService: Provider[NominatimService] + nominatimService: Provider[NominatimService], + tileAggregateService: Provider[TileAggregateService] ) { def comment: CommentService = commentService.get() @@ -98,4 +99,6 @@ class ServiceManager @Inject() ( def leaderboard: LeaderboardService = leaderboardService.get() def nominatim: NominatimService = nominatimService.get() + + def tileAggregate: TileAggregateService = tileAggregateService.get() } diff --git a/app/org/maproulette/framework/service/TileAggregateService.scala b/app/org/maproulette/framework/service/TileAggregateService.scala new file mode 100644 index 000000000..6427168fa --- /dev/null +++ b/app/org/maproulette/framework/service/TileAggregateService.scala @@ -0,0 +1,586 @@ +/* + * Copyright (C) 2020 MapRoulette contributors (see CONTRIBUTORS.md). + * Licensed under the Apache License, Version 2.0 (see LICENSE). + */ + +package org.maproulette.framework.service + +import javax.inject.{Inject, Singleton} +import org.maproulette.framework.model.{ + TaskMarker, + TileAggregate, + ClusterPoint, + TaskMarkerResponse, + TaskClusterSummary, + OverlappingTaskMarker, + Point +} +import org.maproulette.framework.repository.{TileAggregateRepository, TaskClusterRepository} +import org.maproulette.session.SearchLocation +import org.slf4j.LoggerFactory +import play.api.libs.json.Json + +import scala.collection.mutable.ListBuffer + +/** + * Service layer for tile-based task aggregation. + * Provides efficient map display for large datasets by using pre-computed tiles + * with filtering by difficulty and global, recursive drilling for location_id, + * and re-clustering into ~80 clusters. + */ +@Singleton +class TileAggregateService @Inject() ( + repository: TileAggregateRepository, + taskClusterRepository: TaskClusterRepository, + nominatimService: NominatimService +) { + private val logger = LoggerFactory.getLogger(this.getClass) + + // Threshold for switching from clusters to individual tasks + val CLUSTER_THRESHOLD = 2000 + + // Maximum pre-computed zoom level + val MAX_PRECOMPUTED_ZOOM = 14 + + // Target tile range per axis - aim for 8-16 tiles across the viewport + // This gives good granularity for clustering without overwhelming the query + val TARGET_TILES_PER_AXIS = 12 + + // Target number of clusters for final output + val TARGET_CLUSTERS = 80 + + /** + * Get tile data for a bounding box with filtering. + * Supports difficulty, global, location_id, and keywords filters. + * Returns TaskMarkerResponse with clusters, tasks, and overlapping tasks. + * + * @param zoom Zoom level + * @param bounds Bounding box + * @param difficulty Optional difficulty filter + * @param global Include global challenges + * @param locationId Optional Nominatim place_id for polygon filtering + * @param keywords Optional keywords (triggers fallback to dynamic query) + * @return TaskMarkerResponse with clusters or tasks (including overlaps) + */ + def getTileData( + zoom: Int, + bounds: SearchLocation, + difficulty: Option[Int] = None, + global: Boolean = false, + locationId: Option[Long] = None, + keywords: Option[String] = None + ): TaskMarkerResponse = { + + // Keywords filter requires fallback (challenge-level filter, not pre-computed) + if (keywords.exists(_.trim.nonEmpty)) { + return getFallbackData(bounds, difficulty, global, locationId, keywords) + } + + // Calculate optimal query zoom based on viewport size + // We want roughly TARGET_TILES_PER_AXIS tiles across the viewport for good clustering + val effectiveZoom = calculateOptimalZoom(bounds) + + // Collect all data points (either tile centroids or actual tasks) + val collectedPoints = ListBuffer[ClusterPoint]() + val collectedTasks = ListBuffer[TaskMarker]() + + if (locationId.isDefined) { + // Location ID filtering with recursive drilling + processLocationFiltering( + effectiveZoom, + bounds, + difficulty, + global, + locationId.get, + collectedPoints, + collectedTasks + ) + } else { + // Standard tile-based processing + processTiles(effectiveZoom, bounds, difficulty, global, collectedPoints, collectedTasks) + } + + // Calculate total count + val totalCount = collectedPoints.map(_.count).sum + collectedTasks.size + + // If we have few enough tasks, return them with overlap detection + if (totalCount < CLUSTER_THRESHOLD && collectedTasks.nonEmpty) { + val (singleMarkers, overlappingMarkers) = detectOverlaps(collectedTasks.toList) + return TaskMarkerResponse( + totalCount = totalCount, + tasks = Some(singleMarkers), + overlappingTasks = if (overlappingMarkers.nonEmpty) Some(overlappingMarkers) else None, + clusters = None + ) + } + + // Combine tile centroids with task locations for clustering + val allPoints = collectedPoints.toList ++ collectedTasks.map { task => + ClusterPoint(task.location.lat, task.location.lng, 1) + } + + if (allPoints.isEmpty) { + return TaskMarkerResponse(totalCount = 0) + } + + // Re-cluster into ~80 clusters, then merge nearby clusters to prevent visual overlap + val initialClusters = kMeansClustering(allPoints, TARGET_CLUSTERS) + val clusters = mergeNearbyClusters(initialClusters, zoom) + + // Convert ClusterPoints to TaskClusterSummary + val clusterSummaries = clusters.zipWithIndex.map { + case (cp, idx) => + TaskClusterSummary( + clusterId = idx, + numberOfPoints = cp.count, + taskId = None, + taskStatus = None, + point = Point(cp.lat, cp.lng), + bounding = Json.obj() + ) + } + + TaskMarkerResponse( + totalCount = totalCount, + tasks = None, + overlappingTasks = None, + clusters = Some(clusterSummaries) + ) + } + + /** + * Merge clusters that are too close together to prevent visual overlap. + * Minimum distance is calculated based on viewport zoom level. + * At lower zoom levels, clusters need to be farther apart in degrees. + * Iterates until no more merges are possible. + */ + private def mergeNearbyClusters( + clusters: List[ClusterPoint], + viewportZoom: Int + ): List[ClusterPoint] = { + if (clusters.size <= 1) return clusters + + // Calculate minimum distance in degrees based on zoom level + // At zoom 0, world is ~360 degrees wide displayed in ~256 pixels + // We want clusters to be at least ~25 pixels apart visually + // degrees_per_pixel = 360 / (256 * 2^zoom) + // min_distance = pixels * degrees_per_pixel + val pixelBuffer = 25.0 + val minDistanceDeg = pixelBuffer * 360.0 / (256.0 * math.pow(2, viewportZoom)) + + var current = clusters + var changed = true + var maxIters = 50 // Prevent infinite loops + + while (changed && maxIters > 0) { + changed = false + maxIters -= 1 + + val result = ListBuffer[ClusterPoint]() + val used = Array.fill(current.size)(false) + + for (i <- current.indices if !used(i)) { + var merged = current(i) + used(i) = true + + // Find all unused clusters within minimum distance and merge them + for (j <- (i + 1) until current.size if !used(j)) { + val other = current(j) + val dist = distance(merged.lat, merged.lng, other.lat, other.lng) + + if (dist < minDistanceDeg) { + // Merge: weighted average of positions, sum of counts + val totalCount = merged.count + other.count + val newLat = (merged.lat * merged.count + other.lat * other.count) / totalCount + val newLng = (merged.lng * merged.count + other.lng * other.count) / totalCount + merged = ClusterPoint(newLat, newLng, totalCount) + used(j) = true + changed = true + } + } + + result += merged + } + + current = result.toList + } + + current + } + + /** + * Detect overlapping tasks (tasks at the same location within ~0.1 meters) + * Groups tasks by location and separates single vs overlapping markers + */ + private def detectOverlaps( + tasks: List[TaskMarker] + ): (List[TaskMarker], List[OverlappingTaskMarker]) = { + // Group tasks by rounded location (precision of ~0.1 meters = 0.000001 degrees) + val precision = 1000000.0 + val grouped = tasks.groupBy { task => + ( + math.round(task.location.lat * precision), + math.round(task.location.lng * precision) + ) + } + + val singleMarkers = ListBuffer[TaskMarker]() + val overlappingMarkers = ListBuffer[OverlappingTaskMarker]() + + grouped.values.foreach { groupTasks => + if (groupTasks.size == 1) { + singleMarkers += groupTasks.head + } else { + // Use the first task's location as the representative location + val location = groupTasks.head.location + overlappingMarkers += OverlappingTaskMarker(location, groupTasks) + } + } + + (singleMarkers.toList, overlappingMarkers.toList) + } + + /** + * Process tiles in the bounding box using tile centroids. + * Uses a two-phase approach: + * 1. First pass: collect all tile centroids (single query, fast) + * 2. If total count is low enough, fetch actual tasks in a single batched query + */ + private def processTiles( + zoom: Int, + bounds: SearchLocation, + difficulty: Option[Int], + global: Boolean, + collectedPoints: ListBuffer[ClusterPoint], + collectedTasks: ListBuffer[TaskMarker] + ): Unit = { + val tiles = repository.getTilesInBounds(zoom, bounds) + + // First pass: calculate total count and collect tile info + var totalCount = 0 + val tilesWithCounts = tiles.flatMap { tile => + val filteredCount = tile.getFilteredCount(difficulty, global) + if (filteredCount > 0) { + totalCount += filteredCount + Some((tile, filteredCount)) + } else { + None + } + } + + // If total count is low enough, fetch actual tasks in a single batched query + if (totalCount < CLUSTER_THRESHOLD && tilesWithCounts.nonEmpty) { + // Fetch all tasks in the bounding box with a single query (much faster than per-tile) + val tasks = repository.getTaskMarkersInBounds(bounds, difficulty, global) + collectedTasks ++= tasks + } else { + // Use tile centroids for clustering (no additional queries needed) + tilesWithCounts.foreach { + case (tile, filteredCount) => + collectedPoints += ClusterPoint(tile.centroidLat, tile.centroidLng, filteredCount) + } + } + } + + /** + * Process with location_id filtering using tile-based approach. + * Filters pre-computed tiles whose centroids are within the polygon - much faster + * than querying millions of tasks directly, even for complex multipolygons like France. + */ + private def processLocationFiltering( + startZoom: Int, + bounds: SearchLocation, + difficulty: Option[Int], + global: Boolean, + locationId: Long, + collectedPoints: ListBuffer[ClusterPoint], + collectedTasks: ListBuffer[TaskMarker] + ): Unit = { + // Get the location polygon from Nominatim + val locationPolygon = nominatimService.getPolygonByPlaceId(locationId) + + locationPolygon match { + case Some(polygonWkt) => + // Use tile-based filtering: first filter by viewport bounds (fast indexed lookup), + // then check which tile centroids are within the polygon + val tiles = repository.getTilesInPolygon(MAX_PRECOMPUTED_ZOOM, polygonWkt, bounds) + + // Calculate total count from tiles (with difficulty/global filtering) + var totalCount = 0 + val tilesWithCounts = tiles.flatMap { tile => + val filteredCount = tile.getFilteredCount(difficulty, global) + if (filteredCount > 0) { + totalCount += filteredCount + Some((tile, filteredCount)) + } else { + None + } + } + + if (totalCount < CLUSTER_THRESHOLD && totalCount > 0) { + // Low count - fetch actual tasks within the polygon + val tasks = repository.getTaskMarkersInPolygonSimplified( + polygonWkt, difficulty, global, Some(CLUSTER_THRESHOLD) + ) + collectedTasks ++= tasks + } else if (tilesWithCounts.nonEmpty) { + // High count - use tile centroids for clustering + tilesWithCounts.foreach { + case (tile, filteredCount) => + collectedPoints += ClusterPoint(tile.centroidLat, tile.centroidLng, filteredCount) + } + } + + case None => + // Location not found, fall back to standard processing + logger.warn(s"Location polygon not found for place_id: $locationId") + processTiles(startZoom, bounds, difficulty, global, collectedPoints, collectedTasks) + } + } + + /** + * Fallback to dynamic query for keywords filtering. + * Uses same thresholds as tile-based path for consistency. + */ + private def getFallbackData( + bounds: SearchLocation, + difficulty: Option[Int], + global: Boolean, + locationId: Option[Long], + keywords: Option[String] + ): TaskMarkerResponse = { + val boundingBox = bounds + val statusList = List(0, 3, 6) + + val taskCount = taskClusterRepository.queryCountTaskMarkers( + statusList, + global, + boundingBox, + locationId, + keywords, + difficulty + ) + + // Use same threshold as tile-based path for consistency + if (taskCount >= CLUSTER_THRESHOLD) { + // Too many tasks - return clusters + val clusters = taskClusterRepository.queryTaskMarkersClustered( + statusList, + global, + boundingBox, + locationId, + keywords, + difficulty + ) + TaskMarkerResponse(totalCount = taskCount, clusters = Some(clusters)) + } else { + // Few enough tasks - return individual markers with overlap detection + val (singleMarkers, overlappingMarkers) = taskClusterRepository.queryTaskMarkersWithOverlaps( + statusList, + global, + boundingBox, + locationId, + keywords, + difficulty + ) + TaskMarkerResponse( + totalCount = taskCount, + tasks = Some(singleMarkers), + overlappingTasks = if (overlappingMarkers.nonEmpty) Some(overlappingMarkers) else None + ) + } + } + + /** + * Simple k-means clustering implementation. + * Groups points into k clusters and returns cluster centroids with counts. + */ + private def kMeansClustering(points: List[ClusterPoint], k: Int): List[ClusterPoint] = { + if (points.isEmpty) return List.empty + if (points.size <= k) return points + + val numClusters = math.min(k, points.size) + + // Initialize centroids using k-means++ style selection + var centroids = initializeCentroids(points, numClusters) + + // Run k-means iterations + val maxIterations = 20 + var iteration = 0 + var changed = true + + while (iteration < maxIterations && changed) { + // Assign points to nearest centroid + val assignments = points.map { point => + val nearest = centroids.zipWithIndex.minBy { + case (centroid, _) => + distance(point.lat, point.lng, centroid._1, centroid._2) + }._2 + (point, nearest) + } + + // Recalculate centroids + val newCentroids = (0 until numClusters).map { i => + val clusterPoints = assignments.filter(_._2 == i).map(_._1) + if (clusterPoints.isEmpty) { + centroids(i) + } else { + val totalWeight = clusterPoints.map(_.count).sum.toDouble + val avgLat = clusterPoints.map(p => p.lat * p.count).sum / totalWeight + val avgLng = clusterPoints.map(p => p.lng * p.count).sum / totalWeight + (avgLat, avgLng) + } + }.toList + + changed = !newCentroids.equals(centroids) + centroids = newCentroids + iteration += 1 + } + + // Calculate final clusters with counts + val assignments = points.map { point => + val nearest = centroids.zipWithIndex.minBy { + case (centroid, _) => + distance(point.lat, point.lng, centroid._1, centroid._2) + }._2 + (point, nearest) + } + + centroids.zipWithIndex + .map { + case ((lat, lng), i) => + val count = assignments.filter(_._2 == i).map(_._1.count).sum + ClusterPoint(lat, lng, count) + } + .filter(_.count > 0) + } + + /** + * Initialize centroids using k-means++ algorithm for better cluster quality. + * K-means++ selects initial centroids that are well-spread across the data, + * leading to faster convergence and better final clusters. + */ + private def initializeCentroids(points: List[ClusterPoint], k: Int): List[(Double, Double)] = { + if (points.isEmpty || k <= 0) return List.empty + + val random = new scala.util.Random(42) // Fixed seed for reproducibility + val centroids = ListBuffer[(Double, Double)]() + + // Step 1: Choose first centroid uniformly at random (weighted by count) + val totalWeight = points.map(_.count).sum.toDouble + var cumWeight = 0.0 + val targetWeight1 = random.nextDouble() * totalWeight + val firstPoint = points.find { p => + cumWeight += p.count + cumWeight >= targetWeight1 + }.getOrElse(points.head) + centroids += ((firstPoint.lat, firstPoint.lng)) + + // Step 2: Choose remaining centroids with probability proportional to D(x)² + while (centroids.size < k) { + // Calculate squared distance from each point to nearest centroid + val distances = points.map { p => + val minDist = centroids.map { c => + distance(p.lat, p.lng, c._1, c._2) + }.min + // Weight by point count for proper distribution + (p, minDist * minDist * p.count) + } + + // Calculate cumulative distribution + val totalDist = distances.map(_._2).sum + if (totalDist <= 0) { + // All points are at existing centroids, pick remaining randomly + val remaining = points.filterNot { p => + centroids.exists(c => c._1 == p.lat && c._2 == p.lng) + } + if (remaining.nonEmpty) { + val next = remaining(random.nextInt(remaining.size)) + centroids += ((next.lat, next.lng)) + } else { + // Fallback: duplicate a centroid (shouldn't happen in practice) + centroids += centroids.last + } + } else { + // Select next centroid with probability proportional to D(x)² + val target = random.nextDouble() * totalDist + var cumDist = 0.0 + val nextPoint = distances.find { case (_, d) => + cumDist += d + cumDist >= target + }.map(_._1).getOrElse(points.head) + centroids += ((nextPoint.lat, nextPoint.lng)) + } + } + + centroids.toList + } + + private def distance(lat1: Double, lng1: Double, lat2: Double, lng2: Double): Double = { + // Simple Euclidean distance (sufficient for clustering purposes) + val dLat = lat2 - lat1 + val dLng = lng2 - lng1 + math.sqrt(dLat * dLat + dLng * dLng) + } + + /** + * Calculate optimal zoom level for querying tiles based on viewport size. + * Aims for TARGET_TILES_PER_AXIS tiles across the viewport for good clustering + * without overwhelming the query with too many tiles. + */ + private def calculateOptimalZoom(bounds: SearchLocation): Int = { + // Calculate viewport width in degrees (handle anti-meridian crossing) + val lngSpan = if (bounds.left > bounds.right) { + (180.0 - bounds.left) + (bounds.right + 180.0) + } else { + bounds.right - bounds.left + } + val latSpan = bounds.top - bounds.bottom + + // Use the larger span to determine zoom level + val maxSpan = math.max(lngSpan, latSpan) + + // At zoom z, each tile covers 360/2^z degrees of longitude + // We want: maxSpan / (360/2^z) ≈ TARGET_TILES_PER_AXIS + // Solving: 2^z ≈ TARGET_TILES_PER_AXIS * 360 / maxSpan + // z ≈ log2(TARGET_TILES_PER_AXIS * 360 / maxSpan) + val idealZoom = if (maxSpan > 0) { + math.log(TARGET_TILES_PER_AXIS * 360.0 / maxSpan) / math.log(2) + } else { + MAX_PRECOMPUTED_ZOOM.toDouble + } + + // Clamp to valid pre-computed zoom range (0-14) + math.max(0, math.min(MAX_PRECOMPUTED_ZOOM, math.round(idealZoom).toInt)) + } + + /** + * Full rebuild of a specific zoom level + */ + def rebuildZoomLevel(zoom: Int): Int = { + logger.info(s"Starting full rebuild of zoom level $zoom") + val tilesCreated = repository.rebuildZoomLevel(zoom) + logger.info(s"Completed rebuild of zoom level $zoom: $tilesCreated tiles created") + tilesCreated + } + + /** + * Full rebuild of all zoom levels (0-14) + */ + def rebuildAllTiles(): Int = { + logger.info("Starting full rebuild of all tile aggregates") + var totalTiles = 0 + for (zoom <- 0 to 14) { + totalTiles += rebuildZoomLevel(zoom) + } + logger.info(s"Completed full rebuild: $totalTiles total tiles created") + totalTiles + } + + /** + * Get statistics about the tile system + */ + def getStats(): Map[String, Int] = { + Map("totalTiles" -> repository.getTotalTileCount()) + } +} diff --git a/app/org/maproulette/jobs/Scheduler.scala b/app/org/maproulette/jobs/Scheduler.scala index b2e324153..aa3a2fae2 100644 --- a/app/org/maproulette/jobs/Scheduler.scala +++ b/app/org/maproulette/jobs/Scheduler.scala @@ -115,6 +115,13 @@ class Scheduler @Inject() ( Config.KEY_SCHEDULER_UPDATE_CHALLENGE_COMPLETION_INTERVAL ) + schedule( + "refreshTileAggregates", + "Rebuilding tile aggregates", + 1.minute, + Config.KEY_SCHEDULER_TILE_REFRESH_INTERVAL + ) + scheduleAtTime( "sendCountNotificationDailyEmails", "Sending Count Notification Daily Emails", diff --git a/app/org/maproulette/jobs/SchedulerActor.scala b/app/org/maproulette/jobs/SchedulerActor.scala index 28ff6eeee..a6a4a01a1 100644 --- a/app/org/maproulette/jobs/SchedulerActor.scala +++ b/app/org/maproulette/jobs/SchedulerActor.scala @@ -92,6 +92,8 @@ class SchedulerActor @Inject() ( this.handleArchiveChallenges(action) case RunJob("updateChallengeCompletionMetrics", action) => this.handleUpdateChallengeCompletionMetrics(action) + case RunJob("refreshTileAggregates", action) => + this.refreshTileAggregates(action) } /** @@ -876,6 +878,25 @@ class SchedulerActor @Inject() ( logger.warn(s"The KeepRight challenge creation failed. ${f.getMessage}") } } + + /** + * Rebuilds all pre-computed tile aggregates. + * Tiles are used for efficient map display of tasks at scale. + * + * @param action - action string + */ + def refreshTileAggregates(action: String): Unit = { + val start = System.currentTimeMillis + logger.info(s"Scheduled Task '$action': Starting full tile rebuild") + + val totalTiles = serviceManager.tileAggregate.rebuildAllTiles() + + val totalTime = System.currentTimeMillis - start + logger.info( + s"Scheduled Task '$action': Finished run. Time spent: ${totalTime}ms. " + + s"Total tiles: $totalTiles" + ) + } } object SchedulerActor { diff --git a/conf/application.conf b/conf/application.conf index 34a3f34bd..4fac1b065 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -292,6 +292,9 @@ maproulette { interval = "7 days" startTime = "1:00:00" # Snapshot every week at 1am local time } + tileRefresh { + interval = "5 minutes" + } } mapillary { host = "a.mapillary.com" diff --git a/conf/evolutions/default/107.sql b/conf/evolutions/default/107.sql new file mode 100644 index 000000000..42e2e9dc5 --- /dev/null +++ b/conf/evolutions/default/107.sql @@ -0,0 +1,84 @@ +# --- !Ups + +CREATE TABLE tile_aggregates ( + z SMALLINT NOT NULL, + x INTEGER NOT NULL, + y INTEGER NOT NULL, + task_count INTEGER DEFAULT 0, + counts_by_filter JSONB DEFAULT '{}'::jsonb, + centroid_lat DOUBLE PRECISION, + centroid_lng DOUBLE PRECISION, + last_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), + PRIMARY KEY (z, x, y) +);; + +CREATE INDEX idx_tile_aggregates_count ON tile_aggregates (task_count) WHERE task_count > 0;; + +CREATE FUNCTION lng_to_tile_x(lng DOUBLE PRECISION, zoom INTEGER) RETURNS INTEGER AS $$ + SELECT FLOOR((lng + 180.0) / 360.0 * (1 << zoom))::INTEGER +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;; + +CREATE FUNCTION lat_to_tile_y(lat DOUBLE PRECISION, zoom INTEGER) RETURNS INTEGER AS $$ + SELECT FLOOR((1.0 - LN(TAN(RADIANS(GREATEST(-85.0511, LEAST(85.0511, lat)))) + + 1.0 / COS(RADIANS(GREATEST(-85.0511, LEAST(85.0511, lat))))) / PI()) / 2.0 * (1 << zoom))::INTEGER +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;; + +CREATE FUNCTION rebuild_zoom_level(p_zoom INTEGER) RETURNS INTEGER AS $$ +DECLARE + tiles_affected INTEGER;; +BEGIN + DELETE FROM tile_aggregates WHERE z = p_zoom;; + + INSERT INTO tile_aggregates (z, x, y, task_count, counts_by_filter, centroid_lat, centroid_lng) + SELECT + p_zoom, + lng_to_tile_x(ST_X(t.location), p_zoom), + lat_to_tile_y(ST_Y(t.location), p_zoom), + COUNT(*)::INTEGER, + jsonb_build_object( + 'd1_gf', COUNT(*) FILTER (WHERE COALESCE(c.difficulty, 0) = 1 AND NOT COALESCE(c.is_global, false)), + 'd1_gt', COUNT(*) FILTER (WHERE COALESCE(c.difficulty, 0) = 1 AND COALESCE(c.is_global, false)), + 'd2_gf', COUNT(*) FILTER (WHERE COALESCE(c.difficulty, 0) = 2 AND NOT COALESCE(c.is_global, false)), + 'd2_gt', COUNT(*) FILTER (WHERE COALESCE(c.difficulty, 0) = 2 AND COALESCE(c.is_global, false)), + 'd3_gf', COUNT(*) FILTER (WHERE COALESCE(c.difficulty, 0) = 3 AND NOT COALESCE(c.is_global, false)), + 'd3_gt', COUNT(*) FILTER (WHERE COALESCE(c.difficulty, 0) = 3 AND COALESCE(c.is_global, false)), + 'd0_gf', COUNT(*) FILTER (WHERE COALESCE(c.difficulty, 0) NOT IN (1,2,3) AND NOT COALESCE(c.is_global, false)), + 'd0_gt', COUNT(*) FILTER (WHERE COALESCE(c.difficulty, 0) NOT IN (1,2,3) AND COALESCE(c.is_global, false)) + ), + AVG(ST_Y(t.location)), + AVG(ST_X(t.location)) + FROM tasks t + INNER JOIN challenges c ON c.id = t.parent_id + INNER JOIN projects p ON p.id = c.parent_id + WHERE t.location IS NOT NULL + AND NOT ST_IsEmpty(t.location) + AND ST_X(t.location) IS NOT NULL + AND ST_Y(t.location) IS NOT NULL + AND t.status IN (0, 3, 6) + AND c.deleted = FALSE AND c.enabled = TRUE AND c.is_archived = FALSE + AND p.deleted = FALSE AND p.enabled = TRUE + GROUP BY lng_to_tile_x(ST_X(t.location), p_zoom), lat_to_tile_y(ST_Y(t.location), p_zoom);; + + GET DIAGNOSTICS tiles_affected = ROW_COUNT;; + RETURN tiles_affected;; +END +$$ LANGUAGE plpgsql;; + +CREATE FUNCTION rebuild_all_tile_aggregates() RETURNS TABLE(zoom_level INTEGER, tiles_created INTEGER) AS $$ +BEGIN + FOR zoom_level IN 0..14 LOOP + tiles_created := rebuild_zoom_level(zoom_level);; + RETURN NEXT;; + END LOOP;; +END +$$ LANGUAGE plpgsql;; + +-- Run after setup: SELECT * FROM rebuild_all_tile_aggregates(); + +# --- !Downs + +DROP FUNCTION IF EXISTS rebuild_zoom_level(INTEGER);; +DROP FUNCTION IF EXISTS rebuild_all_tile_aggregates();; +DROP FUNCTION IF EXISTS lat_to_tile_y(DOUBLE PRECISION, INTEGER);; +DROP FUNCTION IF EXISTS lng_to_tile_x(DOUBLE PRECISION, INTEGER);; +DROP TABLE IF EXISTS tile_aggregates;; diff --git a/conf/v2_route/task.api b/conf/v2_route/task.api index 9d587ff53..7162f1469 100644 --- a/conf/v2_route/task.api +++ b/conf/v2_route/task.api @@ -678,6 +678,70 @@ PUT /markers/box/:left/:bottom/:right/:top @org.maproulette.framework GET /taskMarkers @org.maproulette.framework.controller.TaskController.getTaskMarkers(statuses: String, global:Boolean ?= false, cluster:Boolean ?= false, bounds: Option[String], location_id: Option[Long] ?= None, keywords:Option[String] ?= None, difficulty:Option[Int] ?= None) ### # tags: [ Task ] +# operationId: task_get_task_tiles +# summary: Get Task Tiles +# description: | +# Returns pre-computed tile aggregates for efficient map display at scale. +# Only includes tasks with status 0 (Created), 3 (Skipped), or 6 (Too Hard). +# For >= 2000 tasks, returns clusters. For < 2000 tasks, returns individual markers. +# +# Filter behavior: +# - difficulty & global: Filtered from pre-computed tile data (fast) +# - location_id: Queries tasks within polygon, clusters if > 2000 +# - keywords: Falls back to dynamic query (not pre-computed) +# responses: +# '200': +# description: Task markers with clusters and/or individual tasks (including overlaps) +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/org.maproulette.framework.model.TaskMarkerResponse' +# parameters: +# - name: z +# in: path +# description: Zoom level (0-14 for pre-computed tiles) +# required: true +# schema: +# type: integer +# minimum: 0 +# maximum: 20 +# - name: bounds +# in: query +# description: Comma-separated bounding box coordinates (left,bottom,right,top) +# required: true +# schema: +# type: string +# example: "-122.5,37.5,-122.0,38.0" +# - name: global +# in: query +# description: Include global challenges (filtered from pre-computed data) +# required: false +# schema: +# type: boolean +# default: false +# - name: location_id +# in: query +# description: Nominatim place_id for polygon filtering +# required: false +# schema: +# type: integer +# - name: keywords +# in: query +# description: Comma-separated keywords to filter by (triggers dynamic query) +# required: false +# schema: +# type: string +# - name: difficulty +# in: query +# description: Filter by difficulty (filtered from pre-computed data) +# required: false +# schema: +# type: integer +# enum: [1, 2, 3] +### +GET /taskTiles/:z @org.maproulette.framework.controller.TaskController.getTaskTiles(z: Int, bounds: String, global: Boolean ?= false, location_id: Option[Long] ?= None, keywords: Option[String] ?= None, difficulty: Option[Int] ?= None) +### +# tags: [ Task ] # operationId: task_update_task_changeset # summary: Update Task Changeset # description: Will update the changeset of the task. It will do this by attempting to match the OSM changeset to the Task based on the geometry and the time that the changeset was executed. diff --git a/test/org/maproulette/utils/TestSpec.scala b/test/org/maproulette/utils/TestSpec.scala index 77dab0cc3..d0f568f34 100644 --- a/test/org/maproulette/utils/TestSpec.scala +++ b/test/org/maproulette/utils/TestSpec.scala @@ -147,6 +147,7 @@ trait TestSpec extends PlaySpec with MockitoSugar { val leaderboardService = mock[LeaderboardService] val taskHistoryService = mock[TaskHistoryService] val nominatimService = mock[NominatimService] + val tileAggregateService = mock[TileAggregateService] val serviceManager = new ServiceManager( Providers.of[ProjectService](projectService), Providers.of[GrantService](grantService), @@ -171,7 +172,8 @@ trait TestSpec extends PlaySpec with MockitoSugar { Providers.of[NotificationService](notificationService), Providers.of[LeaderboardService](leaderboardService), Providers.of[TaskHistoryService](taskHistoryService), - Providers.of[NominatimService](nominatimService) + Providers.of[NominatimService](nominatimService), + Providers.of[TileAggregateService](tileAggregateService) ) val permission = new Permission(Providers.of[DALManager](dalManager), serviceManager, new Config()) From 1322d530898c6f43ede151fe82b70a443dc9e09e Mon Sep 17 00:00:00 2001 From: Collin Beczak Date: Thu, 12 Feb 2026 20:16:19 -0600 Subject: [PATCH 2/6] save --- .../framework/model/TileAggregate.scala | 59 +- .../repository/TileAggregateRepository.scala | 424 +++----------- .../service/TileAggregateService.scala | 550 ++++-------------- app/org/maproulette/jobs/SchedulerActor.scala | 4 +- conf/application.conf | 2 +- conf/evolutions/default/107.sql | 230 ++++++-- 6 files changed, 415 insertions(+), 854 deletions(-) diff --git a/app/org/maproulette/framework/model/TileAggregate.scala b/app/org/maproulette/framework/model/TileAggregate.scala index 97ccbf1f5..328e26a11 100644 --- a/app/org/maproulette/framework/model/TileAggregate.scala +++ b/app/org/maproulette/framework/model/TileAggregate.scala @@ -72,37 +72,64 @@ object FilterCounts { } /** - * Represents a pre-computed tile aggregate for efficient map display at scale. - * Uses Web Mercator tile coordinates (z/x/y). - * Only tracks tasks with status 0, 3, or 6. + * Represents a pre-computed task group at any zoom level (0-14). * - * @param z Zoom level (0-14 for pre-computed) + * Zoom 0-13: One cluster per tile (group_type=2, no task_ids) + * - As zoom increases, tiles get smaller, clusters naturally split + * - Frontend displays these as cluster markers + * + * Zoom 14: One entry per overlap group (group_type=0 or 1, with task_ids) + * - Frontend handles clustering for zoom levels 14-22 + * - Returns individual task markers and overlapping task markers + * + * @param id Database ID + * @param z Zoom level (0-14) * @param x Tile X coordinate * @param y Tile Y coordinate - * @param taskCount Total tasks in this tile (all filters) - * @param countsByFilter Counts broken down by difficulty × global - * @param centroidLat Centroid latitude of all tasks in tile - * @param centroidLng Centroid longitude of all tasks in tile + * @param groupType 0=single task, 1=overlapping tasks, 2=cluster + * @param centroidLat Centroid latitude of the group + * @param centroidLng Centroid longitude of the group + * @param taskIds List of task IDs (empty for clusters at zoom 0-13) + * @param taskCount Number of tasks in this group + * @param countsByFilter Counts broken down by difficulty × global for filtering */ -case class TileAggregate( +case class TileTaskGroup( + id: Long, z: Int, x: Int, y: Int, - taskCount: Int, - countsByFilter: FilterCounts, + groupType: Int, centroidLat: Double, - centroidLng: Double + centroidLng: Double, + taskIds: List[Long], + taskCount: Int, + countsByFilter: FilterCounts ) { /** - * Get the filtered count for this tile based on difficulty and global filters + * Get the filtered count for this group based on difficulty and global filters */ def getFilteredCount(difficulty: Option[Int], global: Boolean): Int = { countsByFilter.getFilteredCount(difficulty, global) } + + /** + * Check if this is a single task (zoom 14 only) + */ + def isSingle: Boolean = groupType == 0 + + /** + * Check if this is an overlapping group (zoom 14 only) + */ + def isOverlapping: Boolean = groupType == 1 + + /** + * Check if this is a cluster (zoom 0-13) + */ + def isCluster: Boolean = groupType == 2 } -object TileAggregate { - implicit val tileAggregateWrites: Writes[TileAggregate] = Json.writes[TileAggregate] - implicit val tileAggregateReads: Reads[TileAggregate] = Json.reads[TileAggregate] +object TileTaskGroup { + implicit val tileTaskGroupWrites: Writes[TileTaskGroup] = Json.writes[TileTaskGroup] + implicit val tileTaskGroupReads: Reads[TileTaskGroup] = Json.reads[TileTaskGroup] } diff --git a/app/org/maproulette/framework/repository/TileAggregateRepository.scala b/app/org/maproulette/framework/repository/TileAggregateRepository.scala index 342f20e91..a8a0923e4 100644 --- a/app/org/maproulette/framework/repository/TileAggregateRepository.scala +++ b/app/org/maproulette/framework/repository/TileAggregateRepository.scala @@ -9,32 +9,37 @@ import java.sql.Connection import anorm._ import anorm.SqlParser.{get, int, double} +import anorm.postgresql._ import javax.inject.{Inject, Singleton} -import org.maproulette.framework.model.{TileAggregate, FilterCounts, TaskMarker, TaskMarkerLocation, ClusterPoint} +import org.maproulette.framework.model.{TileTaskGroup, FilterCounts, TaskMarker, TaskMarkerLocation, ClusterPoint} import org.maproulette.session.SearchLocation import play.api.db.Database import play.api.libs.json.Json /** - * Repository for accessing pre-computed tile aggregates. - * Tiles track tasks with status 0, 3, or 6, with breakdowns by difficulty and global. + * Repository for accessing pre-computed tile task groups. + * All zoom levels (0-14) use overlap detection via ST_ClusterDBSCAN. + * Groups are categorized as single (group_type=0) or overlapping (group_type=1). */ @Singleton class TileAggregateRepository @Inject() (override val db: Database) extends RepositoryMixin { - implicit val baseTable: String = "tile_aggregates" + implicit val baseTable: String = "tile_task_groups" - /** - * Parser for converting database rows to TileAggregate objects - */ - private val tileAggregateParser: RowParser[TileAggregate] = { - get[Int]("z") ~ + // Simplification tolerance in degrees (~1km at equator) + private val SIMPLIFY_TOLERANCE = 0.01 + + private val tileTaskGroupParser: RowParser[TileTaskGroup] = { + get[Long]("id") ~ + get[Int]("z") ~ get[Int]("x") ~ get[Int]("y") ~ + get[Int]("group_type") ~ + get[Double]("centroid_lat") ~ + get[Double]("centroid_lng") ~ + get[List[Long]]("task_ids") ~ get[Int]("task_count") ~ - get[Option[String]]("counts_by_filter") ~ - get[Option[Double]]("centroid_lat") ~ - get[Option[Double]]("centroid_lng") map { - case z ~ x ~ y ~ taskCount ~ countsJson ~ centroidLat ~ centroidLng => + get[Option[String]]("counts_by_filter") map { + case id ~ z ~ x ~ y ~ groupType ~ centroidLat ~ centroidLng ~ taskIds ~ taskCount ~ countsJson => val filterCounts = countsJson .map { json => try { @@ -45,129 +50,98 @@ class TileAggregateRepository @Inject() (override val db: Database) extends Repo } .getOrElse(FilterCounts()) - TileAggregate( - z, - x, - y, - taskCount, - filterCounts, - centroidLat.getOrElse(0.0), - centroidLng.getOrElse(0.0) - ) + TileTaskGroup(id, z, x, y, groupType, centroidLat, centroidLng, taskIds, taskCount, filterCounts) + } + } + + private val clusterPointParser: RowParser[ClusterPoint] = { + get[Double]("lat") ~ + get[Double]("lng") ~ + get[Int]("count") map { + case lat ~ lng ~ count => + ClusterPoint(lat, lng, count) + } + } + + private val taskMarkerParser: RowParser[TaskMarker] = { + get[Long]("id") ~ + get[Double]("lat") ~ + get[Double]("lng") ~ + get[Int]("status") ~ + get[Int]("priority") ~ + get[Option[Long]]("bundle_id") ~ + get[Option[Long]]("locked_by") map { + case id ~ lat ~ lng ~ status ~ priority ~ bundleId ~ lockedBy => + TaskMarker(id, TaskMarkerLocation(lat, lng), status, priority, bundleId, lockedBy) } } /** - * Get tiles for a bounding box at a specific zoom level. + * Get pre-computed task groups in a bounding box at a specific zoom level. * Handles anti-meridian crossing (when bounds.left > bounds.right). */ - def getTilesInBounds( + def getTaskGroupsInBounds( zoom: Int, bounds: SearchLocation - )(implicit c: Option[Connection] = None): List[TileAggregate] = { + )(implicit c: Option[Connection] = None): List[TileTaskGroup] = { this.withMRConnection { implicit c => val minY = latToTileY(bounds.top, zoom) val maxY = latToTileY(bounds.bottom, zoom) - if (bounds.left > bounds.right) { - + // Anti-meridian crossing val leftMinX = lngToTileX(bounds.left, zoom) val leftMaxX = (1 << zoom) - 1 val rightMinX = 0 val rightMaxX = lngToTileX(bounds.right, zoom) - val leftTiles = SQL""" - SELECT z, x, y, task_count, counts_by_filter::text as counts_by_filter, - centroid_lat, centroid_lng - FROM tile_aggregates + val leftGroups = SQL""" + SELECT id, z, x, y, group_type, centroid_lat, centroid_lng, + task_ids, task_count, counts_by_filter::text as counts_by_filter + FROM tile_task_groups WHERE z = $zoom AND x >= $leftMinX AND x <= $leftMaxX AND y >= $minY AND y <= $maxY AND task_count > 0 - """.as(tileAggregateParser.*) + """.as(tileTaskGroupParser.*) - val rightTiles = SQL""" - SELECT z, x, y, task_count, counts_by_filter::text as counts_by_filter, - centroid_lat, centroid_lng - FROM tile_aggregates + val rightGroups = SQL""" + SELECT id, z, x, y, group_type, centroid_lat, centroid_lng, + task_ids, task_count, counts_by_filter::text as counts_by_filter + FROM tile_task_groups WHERE z = $zoom AND x >= $rightMinX AND x <= $rightMaxX AND y >= $minY AND y <= $maxY AND task_count > 0 - """.as(tileAggregateParser.*) + """.as(tileTaskGroupParser.*) - leftTiles ++ rightTiles + leftGroups ++ rightGroups } else { - val minX = lngToTileX(bounds.left, zoom) val maxX = lngToTileX(bounds.right, zoom) SQL""" - SELECT z, x, y, task_count, counts_by_filter::text as counts_by_filter, - centroid_lat, centroid_lng - FROM tile_aggregates + SELECT id, z, x, y, group_type, centroid_lat, centroid_lng, + task_ids, task_count, counts_by_filter::text as counts_by_filter + FROM tile_task_groups WHERE z = $zoom AND x >= $minX AND x <= $maxX AND y >= $minY AND y <= $maxY AND task_count > 0 - """.as(tileAggregateParser.*) + """.as(tileTaskGroupParser.*) } } } /** - * Get a single tile by coordinates - */ - def getTile(z: Int, x: Int, y: Int)( - implicit c: Option[Connection] = None - ): Option[TileAggregate] = { - this.withMRConnection { implicit c => - SQL""" - SELECT z, x, y, task_count, counts_by_filter::text as counts_by_filter, - centroid_lat, centroid_lng - FROM tile_aggregates - WHERE z = $z AND x = $x AND y = $y - """.as(tileAggregateParser.singleOpt) - } - } - - /** - * Get child tiles for recursive drilling (zoom level z+1) + * Get pre-computed task groups within a polygon at a specific zoom level. + * Filters groups whose centroids fall within the polygon. */ - def getChildTiles(z: Int, x: Int, y: Int)( - implicit c: Option[Connection] = None - ): List[TileAggregate] = { - this.withMRConnection { implicit c => - - val childZ = z + 1 - val childXMin = x * 2 - val childXMax = x * 2 + 1 - val childYMin = y * 2 - val childYMax = y * 2 + 1 - - SQL""" - SELECT z, x, y, task_count, counts_by_filter::text as counts_by_filter, - centroid_lat, centroid_lng - FROM tile_aggregates - WHERE z = $childZ - AND x >= $childXMin AND x <= $childXMax - AND y >= $childYMin AND y <= $childYMax - AND task_count > 0 - """.as(tileAggregateParser.*) - } - } - - /** - * Get tiles at a specific zoom level whose centroids fall within a polygon. - * First filters by bounding box (fast indexed lookup), then by polygon containment. - * The polygon is simplified to reduce complexity for faster spatial queries. - */ - def getTilesInPolygon( + def getTaskGroupsInPolygon( zoom: Int, polygonWkt: String, bounds: SearchLocation - )(implicit c: Option[Connection] = None): List[TileAggregate] = { + )(implicit c: Option[Connection] = None): List[TileTaskGroup] = { this.withMRConnection { implicit c => val minX = lngToTileX(bounds.left, zoom) val maxX = lngToTileX(bounds.right, zoom) @@ -178,41 +152,22 @@ class TileAggregateRepository @Inject() (override val db: Database) extends Repo WITH simplified AS ( SELECT ST_Simplify(ST_GeomFromText($polygonWkt, 4326), $SIMPLIFY_TOLERANCE) as geom ) - SELECT z, x, y, task_count, counts_by_filter::text as counts_by_filter, - centroid_lat, centroid_lng - FROM tile_aggregates + SELECT id, z, x, y, group_type, centroid_lat, centroid_lng, + task_ids, task_count, counts_by_filter::text as counts_by_filter + FROM tile_task_groups CROSS JOIN simplified WHERE z = $zoom AND x >= $minX AND x <= $maxX AND y >= $minY AND y <= $maxY AND task_count > 0 AND ST_Contains(simplified.geom, ST_SetSRID(ST_MakePoint(centroid_lng, centroid_lat), 4326)) - """.as(tileAggregateParser.*) + """.as(tileTaskGroupParser.*) } } - /** - * Get tile bounds as (west, south, east, north) - */ - def getTileBounds(z: Int, x: Int, y: Int): (Double, Double, Double, Double) = { - val west = tileToLng(x, z) - val east = tileToLng(x + 1, z) - val north = tileToLat(y, z) - val south = tileToLat(y + 1, z) - (west, south, east, north) - } - - /** - * Get tile bounds as SearchLocation - */ - def getTileBoundsAsSearchLocation(z: Int, x: Int, y: Int): SearchLocation = { - val (west, south, east, north) = getTileBounds(z, x, y) - SearchLocation(west, south, east, north) - } - /** * Fetch all task markers in a bounding box with a single query. - * Much more efficient than querying per-tile when total count is low. + * Used when total count is low enough to return individual tasks. */ def getTaskMarkersInBounds( bounds: SearchLocation, @@ -251,171 +206,6 @@ class TileAggregateRepository @Inject() (override val db: Database) extends Repo } } - /** - * Count tasks within a polygon (for deciding fetch vs cluster strategy) - */ - def countTasksInPolygon( - polygonWkt: String, - difficulty: Option[Int] = None, - global: Boolean = false - )(implicit c: Option[Connection] = None): Int = { - this.withMRConnection { implicit c => - val globalFilter = if (!global) "AND c.is_global = false" else "" - val difficultyFilter = difficulty.map(d => s"AND c.difficulty = $d").getOrElse("") - - SQL""" - SELECT COUNT(DISTINCT tasks.id)::int as count - FROM tasks - INNER JOIN challenges c ON c.id = tasks.parent_id - INNER JOIN projects p ON p.id = c.parent_id - WHERE tasks.location && ST_GeomFromText($polygonWkt, 4326) - AND ST_Intersects(tasks.location, ST_GeomFromText($polygonWkt, 4326)) - AND tasks.status IN (0, 3, 6) - AND c.deleted = false AND c.enabled = true AND c.is_archived = false - AND p.deleted = false AND p.enabled = true - #$globalFilter - #$difficultyFilter - """.as(SqlParser.int("count").single) - } - } - - /** - * Fetch task markers within a polygon (for location_id filtering) - * Uses bounding box operator for index acceleration. - */ - def getTaskMarkersInPolygon( - polygonWkt: String, - difficulty: Option[Int] = None, - global: Boolean = false, - limit: Option[Int] = None - )(implicit c: Option[Connection] = None): List[TaskMarker] = { - this.withMRConnection { implicit c => - val globalFilter = if (!global) "AND c.is_global = false" else "" - val difficultyFilter = difficulty.map(d => s"AND c.difficulty = $d").getOrElse("") - val limitClause = limit.map(l => s"LIMIT $l").getOrElse("") - - SQL""" - SELECT DISTINCT tasks.id, ST_Y(tasks.location) as lat, ST_X(tasks.location) as lng, - tasks.status, tasks.priority, tasks.bundle_id, l.user_id as locked_by - FROM tasks - INNER JOIN challenges c ON c.id = tasks.parent_id - INNER JOIN projects p ON p.id = c.parent_id - LEFT JOIN locked l ON l.item_id = tasks.id AND l.item_type = 2 - WHERE tasks.location && ST_GeomFromText($polygonWkt, 4326) - AND ST_Intersects(tasks.location, ST_GeomFromText($polygonWkt, 4326)) - AND tasks.status IN (0, 3, 6) - AND c.deleted = false AND c.enabled = true AND c.is_archived = false - AND p.deleted = false AND p.enabled = true - #$globalFilter - #$difficultyFilter - #$limitClause - """.as(taskMarkerParser.*) - } - } - - - private val MAX_CLUSTER_POINTS = 50000 - - /** - * Get clustered task markers within a polygon using PostGIS kmeans. - * Returns cluster centroids with counts. - * For very large datasets (>50K points), uses statistical sampling to maintain performance. - */ - def getClusteredTasksInPolygon( - polygonWkt: String, - difficulty: Option[Int] = None, - global: Boolean = false, - numClusters: Int = 80 - )(implicit c: Option[Connection] = None): List[ClusterPoint] = { - this.withMRConnection { implicit c => - val globalFilter = if (!global) "AND c.is_global = false" else "" - val difficultyFilter = difficulty.map(d => s"AND c.difficulty = $d").getOrElse("") - - // Use sampling for very large datasets to prevent slow K-means clustering - // The sample maintains spatial distribution while limiting compute time - SQL""" - WITH task_points AS ( - SELECT tasks.location - FROM tasks - INNER JOIN challenges c ON c.id = tasks.parent_id - INNER JOIN projects p ON p.id = c.parent_id - WHERE tasks.location && ST_GeomFromText($polygonWkt, 4326) - AND ST_Intersects(tasks.location, ST_GeomFromText($polygonWkt, 4326)) - AND tasks.status IN (0, 3, 6) - AND c.deleted = false AND c.enabled = true AND c.is_archived = false - AND p.deleted = false AND p.enabled = true - #$globalFilter - #$difficultyFilter - ), - point_count AS ( - SELECT COUNT(*) as total FROM task_points - ), - sampled_points AS ( - SELECT location - FROM task_points - WHERE (SELECT total FROM point_count) <= $MAX_CLUSTER_POINTS - OR random() < ($MAX_CLUSTER_POINTS::float / GREATEST((SELECT total FROM point_count), 1)) - ), - clustered AS ( - SELECT ST_ClusterKMeans(location, $numClusters) OVER() as cluster_id, location - FROM sampled_points - ) - SELECT - AVG(ST_Y(location)) as lat, - AVG(ST_X(location)) as lng, - COUNT(*)::int as count - FROM clustered - GROUP BY cluster_id - HAVING COUNT(*) > 0 - """.as(clusterPointParser.*) - } - } - - private val clusterPointParser: RowParser[ClusterPoint] = { - get[Double]("lat") ~ - get[Double]("lng") ~ - get[Int]("count") map { - case lat ~ lng ~ count => - ClusterPoint(lat, lng, count) - } - } - - // Simplification tolerance in degrees (~1km at equator) - reduces polygon complexity significantly - private val SIMPLIFY_TOLERANCE = 0.01 - - /** - * Count tasks within a simplified polygon. - * Uses ST_Simplify to reduce polygon complexity for faster queries. - */ - def countTasksInPolygonSimplified( - polygonWkt: String, - difficulty: Option[Int] = None, - global: Boolean = false - )(implicit c: Option[Connection] = None): Int = { - this.withMRConnection { implicit c => - val globalFilter = if (!global) "AND c.is_global = false" else "" - val difficultyFilter = difficulty.map(d => s"AND c.difficulty = $d").getOrElse("") - - SQL""" - WITH simplified AS ( - SELECT ST_Simplify(ST_GeomFromText($polygonWkt, 4326), $SIMPLIFY_TOLERANCE) as geom - ) - SELECT COUNT(DISTINCT tasks.id)::int as count - FROM tasks - CROSS JOIN simplified - INNER JOIN challenges c ON c.id = tasks.parent_id - INNER JOIN projects p ON p.id = c.parent_id - WHERE tasks.location && simplified.geom - AND ST_Intersects(tasks.location, simplified.geom) - AND tasks.status IN (0, 3, 6) - AND c.deleted = false AND c.enabled = true AND c.is_archived = false - AND p.deleted = false AND p.enabled = true - #$globalFilter - #$difficultyFilter - """.as(SqlParser.int("count").single) - } - } - /** * Fetch task markers within a simplified polygon. */ @@ -454,75 +244,7 @@ class TileAggregateRepository @Inject() (override val db: Database) extends Repo } /** - * Get clustered task markers within a simplified polygon. - */ - def getClusteredTasksInPolygonSimplified( - polygonWkt: String, - difficulty: Option[Int] = None, - global: Boolean = false, - numClusters: Int = 80 - )(implicit c: Option[Connection] = None): List[ClusterPoint] = { - this.withMRConnection { implicit c => - val globalFilter = if (!global) "AND c.is_global = false" else "" - val difficultyFilter = difficulty.map(d => s"AND c.difficulty = $d").getOrElse("") - - SQL""" - WITH simplified AS ( - SELECT ST_Simplify(ST_GeomFromText($polygonWkt, 4326), $SIMPLIFY_TOLERANCE) as geom - ), - task_points AS ( - SELECT tasks.location - FROM tasks - CROSS JOIN simplified - INNER JOIN challenges c ON c.id = tasks.parent_id - INNER JOIN projects p ON p.id = c.parent_id - WHERE tasks.location && simplified.geom - AND ST_Intersects(tasks.location, simplified.geom) - AND tasks.status IN (0, 3, 6) - AND c.deleted = false AND c.enabled = true AND c.is_archived = false - AND p.deleted = false AND p.enabled = true - #$globalFilter - #$difficultyFilter - ), - point_count AS ( - SELECT COUNT(*) as total FROM task_points - ), - sampled_points AS ( - SELECT location - FROM task_points - WHERE (SELECT total FROM point_count) <= $MAX_CLUSTER_POINTS - OR random() < ($MAX_CLUSTER_POINTS::float / GREATEST((SELECT total FROM point_count), 1)) - ), - clustered AS ( - SELECT ST_ClusterKMeans(location, $numClusters) OVER() as cluster_id, location - FROM sampled_points - ) - SELECT - AVG(ST_Y(location)) as lat, - AVG(ST_X(location)) as lng, - COUNT(*)::int as count - FROM clustered - GROUP BY cluster_id - HAVING COUNT(*) > 0 - """.as(clusterPointParser.*) - } - } - - private val taskMarkerParser: RowParser[TaskMarker] = { - get[Long]("id") ~ - get[Double]("lat") ~ - get[Double]("lng") ~ - get[Int]("status") ~ - get[Int]("priority") ~ - get[Option[Long]]("bundle_id") ~ - get[Option[Long]]("locked_by") map { - case id ~ lat ~ lng ~ status ~ priority ~ bundleId ~ lockedBy => - TaskMarker(id, TaskMarkerLocation(lat, lng), status, priority, bundleId, lockedBy) - } - } - - /** - * Rebuild all tiles for a specific zoom level + * Rebuild a specific zoom level with overlap detection */ def rebuildZoomLevel(zoom: Int)(implicit c: Option[Connection] = None): Int = { this.withMRTransaction { implicit c => @@ -532,11 +254,11 @@ class TileAggregateRepository @Inject() (override val db: Database) extends Repo } /** - * Get total count of pre-computed tiles + * Get total count of pre-computed task groups */ - def getTotalTileCount()(implicit c: Option[Connection] = None): Int = { + def getTotalTaskGroupCount()(implicit c: Option[Connection] = None): Int = { this.withMRConnection { implicit c => - SQL"SELECT COUNT(*)::int as count FROM tile_aggregates" + SQL"SELECT COUNT(*)::int as count FROM tile_task_groups" .as(SqlParser.int("count").single) } } diff --git a/app/org/maproulette/framework/service/TileAggregateService.scala b/app/org/maproulette/framework/service/TileAggregateService.scala index 6427168fa..6783594ae 100644 --- a/app/org/maproulette/framework/service/TileAggregateService.scala +++ b/app/org/maproulette/framework/service/TileAggregateService.scala @@ -8,7 +8,8 @@ package org.maproulette.framework.service import javax.inject.{Inject, Singleton} import org.maproulette.framework.model.{ TaskMarker, - TileAggregate, + TaskMarkerLocation, + TileTaskGroup, ClusterPoint, TaskMarkerResponse, TaskClusterSummary, @@ -24,9 +25,12 @@ import scala.collection.mutable.ListBuffer /** * Service layer for tile-based task aggregation. - * Provides efficient map display for large datasets by using pre-computed tiles - * with filtering by difficulty and global, recursive drilling for location_id, - * and re-clustering into ~80 clusters. + * + * Zoom 0-13: Returns mix of singles, overlapping markers, and clusters + * - Tiles with 1 task at 1 location → single marker + * - Tiles with N tasks at 1 location → overlapping marker + * - Tiles with tasks at multiple locations → cluster + * Zoom 14+: Returns individual tasks + overlapping markers (frontend handles clustering) */ @Singleton class TileAggregateService @Inject() ( @@ -36,25 +40,13 @@ class TileAggregateService @Inject() ( ) { private val logger = LoggerFactory.getLogger(this.getClass) - // Threshold for switching from clusters to individual tasks - val CLUSTER_THRESHOLD = 2000 - - // Maximum pre-computed zoom level + // Maximum pre-computed zoom level (zoom 14+ all use zoom 14 data) val MAX_PRECOMPUTED_ZOOM = 14 - // Target tile range per axis - aim for 8-16 tiles across the viewport - // This gives good granularity for clustering without overwhelming the query - val TARGET_TILES_PER_AXIS = 12 - - // Target number of clusters for final output - val TARGET_CLUSTERS = 80 - /** * Get tile data for a bounding box with filtering. - * Supports difficulty, global, location_id, and keywords filters. - * Returns TaskMarkerResponse with clusters, tasks, and overlapping tasks. * - * @param zoom Zoom level + * @param zoom Map zoom level (0-22) * @param bounds Bounding box * @param difficulty Optional difficulty filter * @param global Include global challenges @@ -76,271 +68,114 @@ class TileAggregateService @Inject() ( return getFallbackData(bounds, difficulty, global, locationId, keywords) } - // Calculate optimal query zoom based on viewport size - // We want roughly TARGET_TILES_PER_AXIS tiles across the viewport for good clustering - val effectiveZoom = calculateOptimalZoom(bounds) + // Determine which pre-computed zoom level to query + // Zoom 14-22 all use zoom 14 data (frontend clusters) + // Zoom 0-13 use their respective pre-computed clusters + val queryZoom = math.min(zoom, MAX_PRECOMPUTED_ZOOM) - // Collect all data points (either tile centroids or actual tasks) - val collectedPoints = ListBuffer[ClusterPoint]() - val collectedTasks = ListBuffer[TaskMarker]() + // Get polygon for location filtering if needed + val polygonWkt = locationId.flatMap(id => nominatimService.getPolygonByPlaceId(id)) - if (locationId.isDefined) { - // Location ID filtering with recursive drilling - processLocationFiltering( - effectiveZoom, - bounds, - difficulty, - global, - locationId.get, - collectedPoints, - collectedTasks - ) - } else { - // Standard tile-based processing - processTiles(effectiveZoom, bounds, difficulty, global, collectedPoints, collectedTasks) + // Fetch pre-computed groups + val taskGroups = polygonWkt match { + case Some(wkt) => repository.getTaskGroupsInPolygon(queryZoom, wkt, bounds) + case None => repository.getTaskGroupsInBounds(queryZoom, bounds) } - // Calculate total count - val totalCount = collectedPoints.map(_.count).sum + collectedTasks.size - - // If we have few enough tasks, return them with overlap detection - if (totalCount < CLUSTER_THRESHOLD && collectedTasks.nonEmpty) { - val (singleMarkers, overlappingMarkers) = detectOverlaps(collectedTasks.toList) - return TaskMarkerResponse( - totalCount = totalCount, - tasks = Some(singleMarkers), - overlappingTasks = if (overlappingMarkers.nonEmpty) Some(overlappingMarkers) else None, - clusters = None - ) + // Apply difficulty/global filters + val filteredGroups = taskGroups.flatMap { group => + val filteredCount = group.getFilteredCount(difficulty, global) + if (filteredCount > 0) Some((group, filteredCount)) else None } - // Combine tile centroids with task locations for clustering - val allPoints = collectedPoints.toList ++ collectedTasks.map { task => - ClusterPoint(task.location.lat, task.location.lng, 1) - } + val totalCount = filteredGroups.map(_._2).sum - if (allPoints.isEmpty) { + if (totalCount == 0) { return TaskMarkerResponse(totalCount = 0) } - // Re-cluster into ~80 clusters, then merge nearby clusters to prevent visual overlap - val initialClusters = kMeansClustering(allPoints, TARGET_CLUSTERS) - val clusters = mergeNearbyClusters(initialClusters, zoom) - - // Convert ClusterPoints to TaskClusterSummary - val clusterSummaries = clusters.zipWithIndex.map { - case (cp, idx) => - TaskClusterSummary( - clusterId = idx, - numberOfPoints = cp.count, - taskId = None, - taskStatus = None, - point = Point(cp.lat, cp.lng), - bounding = Json.obj() - ) - } - - TaskMarkerResponse( - totalCount = totalCount, - tasks = None, - overlappingTasks = None, - clusters = Some(clusterSummaries) - ) + // Process all groups - separate by type + returnMixedResponse(filteredGroups, totalCount) } /** - * Merge clusters that are too close together to prevent visual overlap. - * Minimum distance is calculated based on viewport zoom level. - * At lower zoom levels, clusters need to be farther apart in degrees. - * Iterates until no more merges are possible. - */ - private def mergeNearbyClusters( - clusters: List[ClusterPoint], - viewportZoom: Int - ): List[ClusterPoint] = { - if (clusters.size <= 1) return clusters - - // Calculate minimum distance in degrees based on zoom level - // At zoom 0, world is ~360 degrees wide displayed in ~256 pixels - // We want clusters to be at least ~25 pixels apart visually - // degrees_per_pixel = 360 / (256 * 2^zoom) - // min_distance = pixels * degrees_per_pixel - val pixelBuffer = 25.0 - val minDistanceDeg = pixelBuffer * 360.0 / (256.0 * math.pow(2, viewportZoom)) - - var current = clusters - var changed = true - var maxIters = 50 // Prevent infinite loops - - while (changed && maxIters > 0) { - changed = false - maxIters -= 1 - - val result = ListBuffer[ClusterPoint]() - val used = Array.fill(current.size)(false) - - for (i <- current.indices if !used(i)) { - var merged = current(i) - used(i) = true - - // Find all unused clusters within minimum distance and merge them - for (j <- (i + 1) until current.size if !used(j)) { - val other = current(j) - val dist = distance(merged.lat, merged.lng, other.lat, other.lng) - - if (dist < minDistanceDeg) { - // Merge: weighted average of positions, sum of counts - val totalCount = merged.count + other.count - val newLat = (merged.lat * merged.count + other.lat * other.count) / totalCount - val newLng = (merged.lng * merged.count + other.lng * other.count) / totalCount - merged = ClusterPoint(newLat, newLng, totalCount) - used(j) = true - changed = true - } - } - - result += merged - } - - current = result.toList - } - - current - } - - /** - * Detect overlapping tasks (tasks at the same location within ~0.1 meters) - * Groups tasks by location and separates single vs overlapping markers + * Return mixed response with clusters, singles, and overlapping markers. + * Works for all zoom levels. + * + * Zoom 0-13: Mostly clusters, but isolated singles/overlaps are included + * Zoom 14+: Only singles and overlapping markers (no clusters) */ - private def detectOverlaps( - tasks: List[TaskMarker] - ): (List[TaskMarker], List[OverlappingTaskMarker]) = { - // Group tasks by rounded location (precision of ~0.1 meters = 0.000001 degrees) - val precision = 1000000.0 - val grouped = tasks.groupBy { task => - ( - math.round(task.location.lat * precision), - math.round(task.location.lng * precision) - ) - } - + private def returnMixedResponse( + groups: List[(TileTaskGroup, Int)], + totalCount: Int + ): TaskMarkerResponse = { val singleMarkers = ListBuffer[TaskMarker]() val overlappingMarkers = ListBuffer[OverlappingTaskMarker]() - - grouped.values.foreach { groupTasks => - if (groupTasks.size == 1) { - singleMarkers += groupTasks.head - } else { - // Use the first task's location as the representative location - val location = groupTasks.head.location - overlappingMarkers += OverlappingTaskMarker(location, groupTasks) - } - } - - (singleMarkers.toList, overlappingMarkers.toList) - } - - /** - * Process tiles in the bounding box using tile centroids. - * Uses a two-phase approach: - * 1. First pass: collect all tile centroids (single query, fast) - * 2. If total count is low enough, fetch actual tasks in a single batched query - */ - private def processTiles( - zoom: Int, - bounds: SearchLocation, - difficulty: Option[Int], - global: Boolean, - collectedPoints: ListBuffer[ClusterPoint], - collectedTasks: ListBuffer[TaskMarker] - ): Unit = { - val tiles = repository.getTilesInBounds(zoom, bounds) - - // First pass: calculate total count and collect tile info - var totalCount = 0 - val tilesWithCounts = tiles.flatMap { tile => - val filteredCount = tile.getFilteredCount(difficulty, global) - if (filteredCount > 0) { - totalCount += filteredCount - Some((tile, filteredCount)) - } else { - None - } - } - - // If total count is low enough, fetch actual tasks in a single batched query - if (totalCount < CLUSTER_THRESHOLD && tilesWithCounts.nonEmpty) { - // Fetch all tasks in the bounding box with a single query (much faster than per-tile) - val tasks = repository.getTaskMarkersInBounds(bounds, difficulty, global) - collectedTasks ++= tasks - } else { - // Use tile centroids for clustering (no additional queries needed) - tilesWithCounts.foreach { - case (tile, filteredCount) => - collectedPoints += ClusterPoint(tile.centroidLat, tile.centroidLng, filteredCount) - } - } - } - - /** - * Process with location_id filtering using tile-based approach. - * Filters pre-computed tiles whose centroids are within the polygon - much faster - * than querying millions of tasks directly, even for complex multipolygons like France. - */ - private def processLocationFiltering( - startZoom: Int, - bounds: SearchLocation, - difficulty: Option[Int], - global: Boolean, - locationId: Long, - collectedPoints: ListBuffer[ClusterPoint], - collectedTasks: ListBuffer[TaskMarker] - ): Unit = { - // Get the location polygon from Nominatim - val locationPolygon = nominatimService.getPolygonByPlaceId(locationId) - - locationPolygon match { - case Some(polygonWkt) => - // Use tile-based filtering: first filter by viewport bounds (fast indexed lookup), - // then check which tile centroids are within the polygon - val tiles = repository.getTilesInPolygon(MAX_PRECOMPUTED_ZOOM, polygonWkt, bounds) - - // Calculate total count from tiles (with difficulty/global filtering) - var totalCount = 0 - val tilesWithCounts = tiles.flatMap { tile => - val filteredCount = tile.getFilteredCount(difficulty, global) - if (filteredCount > 0) { - totalCount += filteredCount - Some((tile, filteredCount)) - } else { - None + val clusterPoints = ListBuffer[(Point, Int)]() + + groups.foreach { + case (group, filteredCount) => + if (group.isSingle) { + // Single task - add as individual marker + group.taskIds.headOption.foreach { taskId => + singleMarkers += TaskMarker( + id = taskId, + location = TaskMarkerLocation(group.centroidLat, group.centroidLng), + status = 0, + priority = 0, + bundleId = None, + lockedBy = None + ) } - } - - if (totalCount < CLUSTER_THRESHOLD && totalCount > 0) { - // Low count - fetch actual tasks within the polygon - val tasks = repository.getTaskMarkersInPolygonSimplified( - polygonWkt, difficulty, global, Some(CLUSTER_THRESHOLD) - ) - collectedTasks ++= tasks - } else if (tilesWithCounts.nonEmpty) { - // High count - use tile centroids for clustering - tilesWithCounts.foreach { - case (tile, filteredCount) => - collectedPoints += ClusterPoint(tile.centroidLat, tile.centroidLng, filteredCount) + } else if (group.isOverlapping) { + // Overlapping tasks - create overlapping marker with all tasks + val tasks = group.taskIds.map { taskId => + TaskMarker( + id = taskId, + location = TaskMarkerLocation(group.centroidLat, group.centroidLng), + status = 0, + priority = 0, + bundleId = None, + lockedBy = None + ) } + overlappingMarkers += OverlappingTaskMarker( + TaskMarkerLocation(group.centroidLat, group.centroidLng), + tasks + ) + } else if (group.isCluster) { + // Cluster - add as cluster point + clusterPoints += ((Point(group.centroidLat, group.centroidLng), filteredCount)) } + } - case None => - // Location not found, fall back to standard processing - logger.warn(s"Location polygon not found for place_id: $locationId") - processTiles(startZoom, bounds, difficulty, global, collectedPoints, collectedTasks) + // Build cluster summaries + val clusterSummaries = if (clusterPoints.nonEmpty) { + Some(clusterPoints.toList.zipWithIndex.map { + case ((point, count), idx) => + TaskClusterSummary( + clusterId = idx, + numberOfPoints = count, + taskId = None, + taskStatus = None, + point = point, + bounding = Json.obj() + ) + }) + } else { + None } + + TaskMarkerResponse( + totalCount = totalCount, + tasks = if (singleMarkers.nonEmpty) Some(singleMarkers.toList) else None, + overlappingTasks = if (overlappingMarkers.nonEmpty) Some(overlappingMarkers.toList) else None, + clusters = clusterSummaries + ) } /** * Fallback to dynamic query for keywords filtering. - * Uses same thresholds as tile-based path for consistency. */ private def getFallbackData( bounds: SearchLocation, @@ -349,36 +184,32 @@ class TileAggregateService @Inject() ( locationId: Option[Long], keywords: Option[String] ): TaskMarkerResponse = { - val boundingBox = bounds - val statusList = List(0, 3, 6) + val statusList = List(0, 3, 6) val taskCount = taskClusterRepository.queryCountTaskMarkers( statusList, global, - boundingBox, + bounds, locationId, keywords, difficulty ) - // Use same threshold as tile-based path for consistency - if (taskCount >= CLUSTER_THRESHOLD) { - // Too many tasks - return clusters + if (taskCount >= 2000) { val clusters = taskClusterRepository.queryTaskMarkersClustered( statusList, global, - boundingBox, + bounds, locationId, keywords, difficulty ) TaskMarkerResponse(totalCount = taskCount, clusters = Some(clusters)) } else { - // Few enough tasks - return individual markers with overlap detection val (singleMarkers, overlappingMarkers) = taskClusterRepository.queryTaskMarkersWithOverlaps( statusList, global, - boundingBox, + bounds, locationId, keywords, difficulty @@ -391,196 +222,33 @@ class TileAggregateService @Inject() ( } } - /** - * Simple k-means clustering implementation. - * Groups points into k clusters and returns cluster centroids with counts. - */ - private def kMeansClustering(points: List[ClusterPoint], k: Int): List[ClusterPoint] = { - if (points.isEmpty) return List.empty - if (points.size <= k) return points - - val numClusters = math.min(k, points.size) - - // Initialize centroids using k-means++ style selection - var centroids = initializeCentroids(points, numClusters) - - // Run k-means iterations - val maxIterations = 20 - var iteration = 0 - var changed = true - - while (iteration < maxIterations && changed) { - // Assign points to nearest centroid - val assignments = points.map { point => - val nearest = centroids.zipWithIndex.minBy { - case (centroid, _) => - distance(point.lat, point.lng, centroid._1, centroid._2) - }._2 - (point, nearest) - } - - // Recalculate centroids - val newCentroids = (0 until numClusters).map { i => - val clusterPoints = assignments.filter(_._2 == i).map(_._1) - if (clusterPoints.isEmpty) { - centroids(i) - } else { - val totalWeight = clusterPoints.map(_.count).sum.toDouble - val avgLat = clusterPoints.map(p => p.lat * p.count).sum / totalWeight - val avgLng = clusterPoints.map(p => p.lng * p.count).sum / totalWeight - (avgLat, avgLng) - } - }.toList - - changed = !newCentroids.equals(centroids) - centroids = newCentroids - iteration += 1 - } - - // Calculate final clusters with counts - val assignments = points.map { point => - val nearest = centroids.zipWithIndex.minBy { - case (centroid, _) => - distance(point.lat, point.lng, centroid._1, centroid._2) - }._2 - (point, nearest) - } - - centroids.zipWithIndex - .map { - case ((lat, lng), i) => - val count = assignments.filter(_._2 == i).map(_._1.count).sum - ClusterPoint(lat, lng, count) - } - .filter(_.count > 0) - } - - /** - * Initialize centroids using k-means++ algorithm for better cluster quality. - * K-means++ selects initial centroids that are well-spread across the data, - * leading to faster convergence and better final clusters. - */ - private def initializeCentroids(points: List[ClusterPoint], k: Int): List[(Double, Double)] = { - if (points.isEmpty || k <= 0) return List.empty - - val random = new scala.util.Random(42) // Fixed seed for reproducibility - val centroids = ListBuffer[(Double, Double)]() - - // Step 1: Choose first centroid uniformly at random (weighted by count) - val totalWeight = points.map(_.count).sum.toDouble - var cumWeight = 0.0 - val targetWeight1 = random.nextDouble() * totalWeight - val firstPoint = points.find { p => - cumWeight += p.count - cumWeight >= targetWeight1 - }.getOrElse(points.head) - centroids += ((firstPoint.lat, firstPoint.lng)) - - // Step 2: Choose remaining centroids with probability proportional to D(x)² - while (centroids.size < k) { - // Calculate squared distance from each point to nearest centroid - val distances = points.map { p => - val minDist = centroids.map { c => - distance(p.lat, p.lng, c._1, c._2) - }.min - // Weight by point count for proper distribution - (p, minDist * minDist * p.count) - } - - // Calculate cumulative distribution - val totalDist = distances.map(_._2).sum - if (totalDist <= 0) { - // All points are at existing centroids, pick remaining randomly - val remaining = points.filterNot { p => - centroids.exists(c => c._1 == p.lat && c._2 == p.lng) - } - if (remaining.nonEmpty) { - val next = remaining(random.nextInt(remaining.size)) - centroids += ((next.lat, next.lng)) - } else { - // Fallback: duplicate a centroid (shouldn't happen in practice) - centroids += centroids.last - } - } else { - // Select next centroid with probability proportional to D(x)² - val target = random.nextDouble() * totalDist - var cumDist = 0.0 - val nextPoint = distances.find { case (_, d) => - cumDist += d - cumDist >= target - }.map(_._1).getOrElse(points.head) - centroids += ((nextPoint.lat, nextPoint.lng)) - } - } - - centroids.toList - } - - private def distance(lat1: Double, lng1: Double, lat2: Double, lng2: Double): Double = { - // Simple Euclidean distance (sufficient for clustering purposes) - val dLat = lat2 - lat1 - val dLng = lng2 - lng1 - math.sqrt(dLat * dLat + dLng * dLng) - } - - /** - * Calculate optimal zoom level for querying tiles based on viewport size. - * Aims for TARGET_TILES_PER_AXIS tiles across the viewport for good clustering - * without overwhelming the query with too many tiles. - */ - private def calculateOptimalZoom(bounds: SearchLocation): Int = { - // Calculate viewport width in degrees (handle anti-meridian crossing) - val lngSpan = if (bounds.left > bounds.right) { - (180.0 - bounds.left) + (bounds.right + 180.0) - } else { - bounds.right - bounds.left - } - val latSpan = bounds.top - bounds.bottom - - // Use the larger span to determine zoom level - val maxSpan = math.max(lngSpan, latSpan) - - // At zoom z, each tile covers 360/2^z degrees of longitude - // We want: maxSpan / (360/2^z) ≈ TARGET_TILES_PER_AXIS - // Solving: 2^z ≈ TARGET_TILES_PER_AXIS * 360 / maxSpan - // z ≈ log2(TARGET_TILES_PER_AXIS * 360 / maxSpan) - val idealZoom = if (maxSpan > 0) { - math.log(TARGET_TILES_PER_AXIS * 360.0 / maxSpan) / math.log(2) - } else { - MAX_PRECOMPUTED_ZOOM.toDouble - } - - // Clamp to valid pre-computed zoom range (0-14) - math.max(0, math.min(MAX_PRECOMPUTED_ZOOM, math.round(idealZoom).toInt)) - } - /** * Full rebuild of a specific zoom level */ def rebuildZoomLevel(zoom: Int): Int = { - logger.info(s"Starting full rebuild of zoom level $zoom") - val tilesCreated = repository.rebuildZoomLevel(zoom) - logger.info(s"Completed rebuild of zoom level $zoom: $tilesCreated tiles created") - tilesCreated + logger.info(s"Starting rebuild of zoom level $zoom") + val groupsCreated = repository.rebuildZoomLevel(zoom) + logger.info(s"Completed rebuild of zoom level $zoom: $groupsCreated groups created") + groupsCreated } /** * Full rebuild of all zoom levels (0-14) */ def rebuildAllTiles(): Int = { - logger.info("Starting full rebuild of all tile aggregates") - var totalTiles = 0 - for (zoom <- 0 to 14) { - totalTiles += rebuildZoomLevel(zoom) + logger.info("Starting full rebuild of all tile task groups") + var totalGroups = 0 + for (zoom <- 0 to MAX_PRECOMPUTED_ZOOM) { + totalGroups += rebuildZoomLevel(zoom) } - logger.info(s"Completed full rebuild: $totalTiles total tiles created") - totalTiles + logger.info(s"Completed full rebuild: $totalGroups total groups created") + totalGroups } /** * Get statistics about the tile system */ def getStats(): Map[String, Int] = { - Map("totalTiles" -> repository.getTotalTileCount()) + Map("totalTaskGroups" -> repository.getTotalTaskGroupCount()) } } diff --git a/app/org/maproulette/jobs/SchedulerActor.scala b/app/org/maproulette/jobs/SchedulerActor.scala index a6a4a01a1..837213fd3 100644 --- a/app/org/maproulette/jobs/SchedulerActor.scala +++ b/app/org/maproulette/jobs/SchedulerActor.scala @@ -170,11 +170,11 @@ class SchedulerActor @Inject() ( db.withTransaction { implicit c => val query = - s"""UPDATE challenges + s"""UPDATE challenges SET location = (SELECT ST_Centroid(ST_Collect(ST_Makevalid(location))) FROM tasks WHERE parent_id = ${id}), - bounding = (SELECT ST_Envelope(ST_Buffer((ST_SetSRID(ST_Extent(location), 4326))::geography,2)::geometry) + bounding = (SELECT ST_Envelope(ST_Expand(ST_SetSRID(ST_Extent(location), 4326), 0.0001)) FROM tasks WHERE parent_id = ${id}), last_updated = NOW(), diff --git a/conf/application.conf b/conf/application.conf index 4fac1b065..710b1148b 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -229,7 +229,7 @@ maproulette { cleanLocks.interval = "1 hour" cleanClaimLocks.interval = "1 hour" runChallengeSchedules.interval = "24 hours" - updateLocations.interval = "2 hours" + # updateLocations.interval = "2 hours" # Disabled for testing cleanOldTasks { interval = "24 hours" olderThan = "31 days" diff --git a/conf/evolutions/default/107.sql b/conf/evolutions/default/107.sql index 42e2e9dc5..04cad0766 100644 --- a/conf/evolutions/default/107.sql +++ b/conf/evolutions/default/107.sql @@ -1,19 +1,31 @@ # --- !Ups -CREATE TABLE tile_aggregates ( +-- Table for storing pre-computed task groups at all zoom levels (0-14) +-- Zoom 0-13: One entry per tile - can be single, overlapping, or cluster +-- - Single (1 task at 1 location): group_type=0, has task_ids +-- - Overlapping (N tasks at 1 location): group_type=1, has task_ids +-- - Cluster (N tasks at multiple locations): group_type=2, no task_ids +-- Zoom 14: One entry per overlap group (for frontend clustering at 14-22) +-- - Single: group_type=0, has task_ids +-- - Overlapping: group_type=1, has task_ids +CREATE TABLE tile_task_groups ( + id SERIAL PRIMARY KEY, z SMALLINT NOT NULL, x INTEGER NOT NULL, y INTEGER NOT NULL, - task_count INTEGER DEFAULT 0, + group_type SMALLINT NOT NULL, -- 0=single task, 1=overlapping tasks, 2=cluster + centroid_lat DOUBLE PRECISION NOT NULL, + centroid_lng DOUBLE PRECISION NOT NULL, + task_ids BIGINT[] NOT NULL, + task_count INTEGER NOT NULL, counts_by_filter JSONB DEFAULT '{}'::jsonb, - centroid_lat DOUBLE PRECISION, - centroid_lng DOUBLE PRECISION, - last_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), - PRIMARY KEY (z, x, y) + last_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW() );; -CREATE INDEX idx_tile_aggregates_count ON tile_aggregates (task_count) WHERE task_count > 0;; +CREATE INDEX idx_tile_task_groups_coords ON tile_task_groups (z, x, y);; +CREATE INDEX idx_tile_task_groups_zoom ON tile_task_groups (z) WHERE task_count > 0;; +-- Coordinate conversion functions CREATE FUNCTION lng_to_tile_x(lng DOUBLE PRECISION, zoom INTEGER) RETURNS INTEGER AS $$ SELECT FLOOR((lng + 180.0) / 360.0 * (1 << zoom))::INTEGER $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;; @@ -23,53 +35,183 @@ CREATE FUNCTION lat_to_tile_y(lat DOUBLE PRECISION, zoom INTEGER) RETURNS INTEGE 1.0 / COS(RADIANS(GREATEST(-85.0511, LEAST(85.0511, lat))))) / PI()) / 2.0 * (1 << zoom))::INTEGER $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;; +-- Function to rebuild a specific zoom level CREATE FUNCTION rebuild_zoom_level(p_zoom INTEGER) RETURNS INTEGER AS $$ DECLARE - tiles_affected INTEGER;; + groups_created INTEGER;; + start_time TIMESTAMP;; + deleted_count INTEGER;; BEGIN - DELETE FROM tile_aggregates WHERE z = p_zoom;; - - INSERT INTO tile_aggregates (z, x, y, task_count, counts_by_filter, centroid_lat, centroid_lng) - SELECT - p_zoom, - lng_to_tile_x(ST_X(t.location), p_zoom), - lat_to_tile_y(ST_Y(t.location), p_zoom), - COUNT(*)::INTEGER, - jsonb_build_object( - 'd1_gf', COUNT(*) FILTER (WHERE COALESCE(c.difficulty, 0) = 1 AND NOT COALESCE(c.is_global, false)), - 'd1_gt', COUNT(*) FILTER (WHERE COALESCE(c.difficulty, 0) = 1 AND COALESCE(c.is_global, false)), - 'd2_gf', COUNT(*) FILTER (WHERE COALESCE(c.difficulty, 0) = 2 AND NOT COALESCE(c.is_global, false)), - 'd2_gt', COUNT(*) FILTER (WHERE COALESCE(c.difficulty, 0) = 2 AND COALESCE(c.is_global, false)), - 'd3_gf', COUNT(*) FILTER (WHERE COALESCE(c.difficulty, 0) = 3 AND NOT COALESCE(c.is_global, false)), - 'd3_gt', COUNT(*) FILTER (WHERE COALESCE(c.difficulty, 0) = 3 AND COALESCE(c.is_global, false)), - 'd0_gf', COUNT(*) FILTER (WHERE COALESCE(c.difficulty, 0) NOT IN (1,2,3) AND NOT COALESCE(c.is_global, false)), - 'd0_gt', COUNT(*) FILTER (WHERE COALESCE(c.difficulty, 0) NOT IN (1,2,3) AND COALESCE(c.is_global, false)) + start_time := clock_timestamp();; + RAISE NOTICE 'Zoom %: Starting rebuild...', p_zoom;; + + DELETE FROM tile_task_groups WHERE z = p_zoom;; + GET DIAGNOSTICS deleted_count = ROW_COUNT;; + RAISE NOTICE 'Zoom %: Deleted % existing groups', p_zoom, deleted_count;; + + IF p_zoom = 14 THEN + -- Zoom 14: Full detail - one entry per overlap group with task_ids + -- Frontend will handle clustering for zoom levels 14-22 + INSERT INTO tile_task_groups (z, x, y, group_type, centroid_lat, centroid_lng, task_ids, task_count, counts_by_filter) + WITH task_clusters AS ( + SELECT + t.id as task_id, + ST_Y(t.location) as lat, + ST_X(t.location) as lng, + lng_to_tile_x(ST_X(t.location), p_zoom) as tile_x, + lat_to_tile_y(ST_Y(t.location), p_zoom) as tile_y, + COALESCE(c.difficulty, 0) as difficulty, + COALESCE(c.is_global, false) as is_global, + ST_ClusterDBSCAN(t.location, eps := 0.000001, minpoints := 1) OVER ( + PARTITION BY lng_to_tile_x(ST_X(t.location), p_zoom), lat_to_tile_y(ST_Y(t.location), p_zoom) + ) as cluster_id + FROM tasks t + INNER JOIN challenges c ON c.id = t.parent_id + INNER JOIN projects p ON p.id = c.parent_id + WHERE t.location IS NOT NULL + AND NOT ST_IsEmpty(t.location) + AND ST_X(t.location) IS NOT NULL + AND ST_Y(t.location) IS NOT NULL + AND t.status IN (0, 3, 6) + AND c.deleted = FALSE AND c.enabled = TRUE AND c.is_archived = FALSE + AND p.deleted = FALSE AND p.enabled = TRUE + ), + grouped_tasks AS ( + SELECT + tile_x, + tile_y, + cluster_id, + AVG(lat) as centroid_lat, + AVG(lng) as centroid_lng, + ARRAY_AGG(task_id ORDER BY task_id) as task_ids, + COUNT(*)::INTEGER as task_count, + jsonb_build_object( + 'd1_gf', COUNT(*) FILTER (WHERE difficulty = 1 AND NOT is_global), + 'd1_gt', COUNT(*) FILTER (WHERE difficulty = 1 AND is_global), + 'd2_gf', COUNT(*) FILTER (WHERE difficulty = 2 AND NOT is_global), + 'd2_gt', COUNT(*) FILTER (WHERE difficulty = 2 AND is_global), + 'd3_gf', COUNT(*) FILTER (WHERE difficulty = 3 AND NOT is_global), + 'd3_gt', COUNT(*) FILTER (WHERE difficulty = 3 AND is_global), + 'd0_gf', COUNT(*) FILTER (WHERE difficulty NOT IN (1,2,3) AND NOT is_global), + 'd0_gt', COUNT(*) FILTER (WHERE difficulty NOT IN (1,2,3) AND is_global) + ) as counts_by_filter + FROM task_clusters + GROUP BY tile_x, tile_y, cluster_id + ) + SELECT + p_zoom as z, + tile_x as x, + tile_y as y, + CASE WHEN task_count = 1 THEN 0 ELSE 1 END as group_type, -- 0=single, 1=overlapping + centroid_lat, + centroid_lng, + task_ids, + task_count, + counts_by_filter + FROM grouped_tasks;; + + GET DIAGNOSTICS groups_created = ROW_COUNT;; + RAISE NOTICE 'Zoom %: Created % task/overlap groups (full detail for frontend clustering)', p_zoom, groups_created;; + ELSE + -- Zoom 0-13: One entry per tile + -- Detect if tile has one location (single/overlap) or multiple locations (cluster) + INSERT INTO tile_task_groups (z, x, y, group_type, centroid_lat, centroid_lng, task_ids, task_count, counts_by_filter) + WITH tile_tasks AS ( + SELECT + t.id as task_id, + ST_Y(t.location) as lat, + ST_X(t.location) as lng, + lng_to_tile_x(ST_X(t.location), p_zoom) as tile_x, + lat_to_tile_y(ST_Y(t.location), p_zoom) as tile_y, + COALESCE(c.difficulty, 0) as difficulty, + COALESCE(c.is_global, false) as is_global, + -- Detect overlaps within the tile (~0.1 meter precision) + ST_ClusterDBSCAN(t.location, eps := 0.000001, minpoints := 1) OVER ( + PARTITION BY lng_to_tile_x(ST_X(t.location), p_zoom), lat_to_tile_y(ST_Y(t.location), p_zoom) + ) as overlap_cluster_id + FROM tasks t + INNER JOIN challenges c ON c.id = t.parent_id + INNER JOIN projects p ON p.id = c.parent_id + WHERE t.location IS NOT NULL + AND NOT ST_IsEmpty(t.location) + AND ST_X(t.location) IS NOT NULL + AND ST_Y(t.location) IS NOT NULL + AND t.status IN (0, 3, 6) + AND c.deleted = FALSE AND c.enabled = TRUE AND c.is_archived = FALSE + AND p.deleted = FALSE AND p.enabled = TRUE ), - AVG(ST_Y(t.location)), - AVG(ST_X(t.location)) - FROM tasks t - INNER JOIN challenges c ON c.id = t.parent_id - INNER JOIN projects p ON p.id = c.parent_id - WHERE t.location IS NOT NULL - AND NOT ST_IsEmpty(t.location) - AND ST_X(t.location) IS NOT NULL - AND ST_Y(t.location) IS NOT NULL - AND t.status IN (0, 3, 6) - AND c.deleted = FALSE AND c.enabled = TRUE AND c.is_archived = FALSE - AND p.deleted = FALSE AND p.enabled = TRUE - GROUP BY lng_to_tile_x(ST_X(t.location), p_zoom), lat_to_tile_y(ST_Y(t.location), p_zoom);; - - GET DIAGNOSTICS tiles_affected = ROW_COUNT;; - RETURN tiles_affected;; + tile_summary AS ( + SELECT + tile_x, + tile_y, + COUNT(DISTINCT overlap_cluster_id)::INTEGER as num_locations, + AVG(lat) as centroid_lat, + AVG(lng) as centroid_lng, + ARRAY_AGG(task_id ORDER BY task_id) as task_ids, + COUNT(*)::INTEGER as task_count, + jsonb_build_object( + 'd1_gf', COUNT(*) FILTER (WHERE difficulty = 1 AND NOT is_global), + 'd1_gt', COUNT(*) FILTER (WHERE difficulty = 1 AND is_global), + 'd2_gf', COUNT(*) FILTER (WHERE difficulty = 2 AND NOT is_global), + 'd2_gt', COUNT(*) FILTER (WHERE difficulty = 2 AND is_global), + 'd3_gf', COUNT(*) FILTER (WHERE difficulty = 3 AND NOT is_global), + 'd3_gt', COUNT(*) FILTER (WHERE difficulty = 3 AND is_global), + 'd0_gf', COUNT(*) FILTER (WHERE difficulty NOT IN (1,2,3) AND NOT is_global), + 'd0_gt', COUNT(*) FILTER (WHERE difficulty NOT IN (1,2,3) AND is_global) + ) as counts_by_filter + FROM tile_tasks + GROUP BY tile_x, tile_y + ) + SELECT + p_zoom as z, + tile_x as x, + tile_y as y, + CASE + WHEN num_locations = 1 AND task_count = 1 THEN 0 -- single task (isolated) + WHEN num_locations = 1 THEN 1 -- overlapping tasks (same location) + ELSE 2 -- cluster (multiple locations) + END as group_type, + centroid_lat, + centroid_lng, + CASE + WHEN num_locations = 1 THEN task_ids -- single/overlap: store task_ids + ELSE ARRAY[]::BIGINT[] -- cluster: no task_ids + END as task_ids, + task_count, + counts_by_filter + FROM tile_summary;; + + GET DIAGNOSTICS groups_created = ROW_COUNT;; + RAISE NOTICE 'Zoom %: Created % tile entries (singles/overlaps/clusters)', p_zoom, groups_created;; + END IF;; + + RAISE NOTICE 'Zoom %: Completed in % ms', p_zoom, EXTRACT(MILLISECONDS FROM (clock_timestamp() - start_time))::INTEGER;; + RETURN groups_created;; END $$ LANGUAGE plpgsql;; +-- Function to rebuild all zoom levels CREATE FUNCTION rebuild_all_tile_aggregates() RETURNS TABLE(zoom_level INTEGER, tiles_created INTEGER) AS $$ +DECLARE + total_start TIMESTAMP;; + total_groups INTEGER := 0;; BEGIN + total_start := clock_timestamp();; + RAISE NOTICE '=== Starting full tile aggregate rebuild ===';; + RAISE NOTICE 'Zoom 0-13: Singles, overlaps, or clusters per tile';; + RAISE NOTICE 'Zoom 14: Individual tasks + overlaps (for frontend clustering at 14-22)';; + RAISE NOTICE '';; + FOR zoom_level IN 0..14 LOOP tiles_created := rebuild_zoom_level(zoom_level);; + total_groups := total_groups + tiles_created;; RETURN NEXT;; END LOOP;; + + RAISE NOTICE '';; + RAISE NOTICE '=== Rebuild complete ===';; + RAISE NOTICE 'Total groups created: %', total_groups;; + RAISE NOTICE 'Total time: % ms', EXTRACT(MILLISECONDS FROM (clock_timestamp() - total_start))::INTEGER;; END $$ LANGUAGE plpgsql;; @@ -77,8 +219,10 @@ $$ LANGUAGE plpgsql;; # --- !Downs -DROP FUNCTION IF EXISTS rebuild_zoom_level(INTEGER);; DROP FUNCTION IF EXISTS rebuild_all_tile_aggregates();; +DROP FUNCTION IF EXISTS rebuild_zoom_level(INTEGER);; DROP FUNCTION IF EXISTS lat_to_tile_y(DOUBLE PRECISION, INTEGER);; DROP FUNCTION IF EXISTS lng_to_tile_x(DOUBLE PRECISION, INTEGER);; -DROP TABLE IF EXISTS tile_aggregates;; +DROP INDEX IF EXISTS idx_tile_task_groups_zoom;; +DROP INDEX IF EXISTS idx_tile_task_groups_coords;; +DROP TABLE IF EXISTS tile_task_groups;; From f7fb6cfbbc3388d82a4b9a49ada08ac499707106 Mon Sep 17 00:00:00 2001 From: Collin Beczak Date: Tue, 17 Feb 2026 12:51:04 -0600 Subject: [PATCH 3/6] Refactor tile coordinate calculations to use effective zoom level - Updated `TileAggregateRepository` to incorporate an effective zoom level for tile calculations, enhancing precision for zoom levels below 14. - Adjusted SQL functions in migration script to align with the new effective zoom logic, ensuring consistent tile coordinate generation across the application. --- .../repository/TileAggregateRepository.scala | 20 ++++++++++++++----- conf/evolutions/default/107.sql | 9 ++++++--- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/app/org/maproulette/framework/repository/TileAggregateRepository.scala b/app/org/maproulette/framework/repository/TileAggregateRepository.scala index a8a0923e4..cb96ab3a7 100644 --- a/app/org/maproulette/framework/repository/TileAggregateRepository.scala +++ b/app/org/maproulette/framework/repository/TileAggregateRepository.scala @@ -90,8 +90,9 @@ class TileAggregateRepository @Inject() (override val db: Database) extends Repo if (bounds.left > bounds.right) { // Anti-meridian crossing + val effectiveZoom = if (zoom < 14) zoom + ZOOM_OFFSET else zoom val leftMinX = lngToTileX(bounds.left, zoom) - val leftMaxX = (1 << zoom) - 1 + val leftMaxX = (1 << effectiveZoom) - 1 val rightMinX = 0 val rightMaxX = lngToTileX(bounds.right, zoom) @@ -263,27 +264,36 @@ class TileAggregateRepository @Inject() (override val db: Database) extends Repo } } + // Offset for tile coordinate calculations (zoom 0 uses zoom 2 grid, etc.) + // Must match the offset in rebuild_zoom_level() SQL function + private val ZOOM_OFFSET = 2 + // Web Mercator coordinate conversion functions + // For zoom 0-13, we use effectiveZoom = zoom + ZOOM_OFFSET for more granular tiles def lngToTileX(lng: Double, zoom: Int): Int = { - math.floor((lng + 180.0) / 360.0 * (1 << zoom)).toInt + val effectiveZoom = if (zoom < 14) zoom + ZOOM_OFFSET else zoom + math.floor((lng + 180.0) / 360.0 * (1 << effectiveZoom)).toInt } def latToTileY(lat: Double, zoom: Int): Int = { + val effectiveZoom = if (zoom < 14) zoom + ZOOM_OFFSET else zoom val latClamped = math.max(-85.0511, math.min(85.0511, lat)) val latRad = math.toRadians(latClamped) math .floor( - (1.0 - math.log(math.tan(latRad) + 1.0 / math.cos(latRad)) / math.Pi) / 2.0 * (1 << zoom) + (1.0 - math.log(math.tan(latRad) + 1.0 / math.cos(latRad)) / math.Pi) / 2.0 * (1 << effectiveZoom) ) .toInt } def tileToLng(x: Int, zoom: Int): Double = { - x.toDouble / (1 << zoom) * 360.0 - 180.0 + val effectiveZoom = if (zoom < 14) zoom + ZOOM_OFFSET else zoom + x.toDouble / (1 << effectiveZoom) * 360.0 - 180.0 } def tileToLat(y: Int, zoom: Int): Double = { - val n = math.Pi - 2.0 * math.Pi * y.toDouble / (1 << zoom) + val effectiveZoom = if (zoom < 14) zoom + ZOOM_OFFSET else zoom + val n = math.Pi - 2.0 * math.Pi * y.toDouble / (1 << effectiveZoom) math.toDegrees(math.atan(math.sinh(n))) } } diff --git a/conf/evolutions/default/107.sql b/conf/evolutions/default/107.sql index 04cad0766..8e694115d 100644 --- a/conf/evolutions/default/107.sql +++ b/conf/evolutions/default/107.sql @@ -41,7 +41,10 @@ DECLARE groups_created INTEGER;; start_time TIMESTAMP;; deleted_count INTEGER;; + effective_zoom INTEGER;; BEGIN + -- Offset zoom by +2 for tile calculations (zoom 0 uses zoom 2 grid, etc.) + effective_zoom := p_zoom + 2;; start_time := clock_timestamp();; RAISE NOTICE 'Zoom %: Starting rebuild...', p_zoom;; @@ -121,13 +124,13 @@ BEGIN t.id as task_id, ST_Y(t.location) as lat, ST_X(t.location) as lng, - lng_to_tile_x(ST_X(t.location), p_zoom) as tile_x, - lat_to_tile_y(ST_Y(t.location), p_zoom) as tile_y, + lng_to_tile_x(ST_X(t.location), effective_zoom) as tile_x, + lat_to_tile_y(ST_Y(t.location), effective_zoom) as tile_y, COALESCE(c.difficulty, 0) as difficulty, COALESCE(c.is_global, false) as is_global, -- Detect overlaps within the tile (~0.1 meter precision) ST_ClusterDBSCAN(t.location, eps := 0.000001, minpoints := 1) OVER ( - PARTITION BY lng_to_tile_x(ST_X(t.location), p_zoom), lat_to_tile_y(ST_Y(t.location), p_zoom) + PARTITION BY lng_to_tile_x(ST_X(t.location), effective_zoom), lat_to_tile_y(ST_Y(t.location), effective_zoom) ) as overlap_cluster_id FROM tasks t INNER JOIN challenges c ON c.id = t.parent_id From c58f050e5b4e641a6c1561aa29dd6bd756909f86 Mon Sep 17 00:00:00 2001 From: Collin Beczak Date: Tue, 17 Feb 2026 12:54:25 -0600 Subject: [PATCH 4/6] run formatting --- .../framework/controller/TaskController.scala | 29 +++++++------- .../repository/TileAggregateRepository.scala | 38 ++++++++++++++----- .../framework/service/NominatimService.scala | 8 +--- 3 files changed, 44 insertions(+), 31 deletions(-) diff --git a/app/org/maproulette/framework/controller/TaskController.scala b/app/org/maproulette/framework/controller/TaskController.scala index cdaefd8f8..2907818a9 100644 --- a/app/org/maproulette/framework/controller/TaskController.scala +++ b/app/org/maproulette/framework/controller/TaskController.scala @@ -363,22 +363,23 @@ class TaskController @Inject() ( val validDifficulty = difficulty.filter(d => d >= 1 && d <= 3) // Parse and validate bounding box - val boundingBox = try { - bounds.split(",").map(_.trim.toDouble).toList match { - case List(left, bottom, right, top) => - // Clamp coordinates to valid ranges - val clampedLeft = math.max(-180.0, math.min(180.0, left)) - val clampedRight = math.max(-180.0, math.min(180.0, right)) - val clampedBottom = math.max(-90.0, math.min(90.0, bottom)) - val clampedTop = math.max(-90.0, math.min(90.0, top)) - SearchLocation(clampedLeft, clampedBottom, clampedRight, clampedTop) - case _ => + val boundingBox = + try { + bounds.split(",").map(_.trim.toDouble).toList match { + case List(left, bottom, right, top) => + // Clamp coordinates to valid ranges + val clampedLeft = math.max(-180.0, math.min(180.0, left)) + val clampedRight = math.max(-180.0, math.min(180.0, right)) + val clampedBottom = math.max(-90.0, math.min(90.0, bottom)) + val clampedTop = math.max(-90.0, math.min(90.0, top)) + SearchLocation(clampedLeft, clampedBottom, clampedRight, clampedTop) + case _ => + SearchLocation(-180.0, -90.0, 180.0, 90.0) + } + } catch { + case _: NumberFormatException => SearchLocation(-180.0, -90.0, 180.0, 90.0) } - } catch { - case _: NumberFormatException => - SearchLocation(-180.0, -90.0, 180.0, 90.0) - } val response = this.serviceManager.tileAggregate.getTileData( validZoom, diff --git a/app/org/maproulette/framework/repository/TileAggregateRepository.scala b/app/org/maproulette/framework/repository/TileAggregateRepository.scala index cb96ab3a7..7d653bad2 100644 --- a/app/org/maproulette/framework/repository/TileAggregateRepository.scala +++ b/app/org/maproulette/framework/repository/TileAggregateRepository.scala @@ -11,7 +11,13 @@ import anorm._ import anorm.SqlParser.{get, int, double} import anorm.postgresql._ import javax.inject.{Inject, Singleton} -import org.maproulette.framework.model.{TileTaskGroup, FilterCounts, TaskMarker, TaskMarkerLocation, ClusterPoint} +import org.maproulette.framework.model.{ + TileTaskGroup, + FilterCounts, + TaskMarker, + TaskMarkerLocation, + ClusterPoint +} import org.maproulette.session.SearchLocation import play.api.db.Database import play.api.libs.json.Json @@ -50,7 +56,18 @@ class TileAggregateRepository @Inject() (override val db: Database) extends Repo } .getOrElse(FilterCounts()) - TileTaskGroup(id, z, x, y, groupType, centroidLat, centroidLng, taskIds, taskCount, filterCounts) + TileTaskGroup( + id, + z, + x, + y, + groupType, + centroidLat, + centroidLng, + taskIds, + taskCount, + filterCounts + ) } } @@ -91,10 +108,10 @@ class TileAggregateRepository @Inject() (override val db: Database) extends Repo if (bounds.left > bounds.right) { // Anti-meridian crossing val effectiveZoom = if (zoom < 14) zoom + ZOOM_OFFSET else zoom - val leftMinX = lngToTileX(bounds.left, zoom) - val leftMaxX = (1 << effectiveZoom) - 1 - val rightMinX = 0 - val rightMaxX = lngToTileX(bounds.right, zoom) + val leftMinX = lngToTileX(bounds.left, zoom) + val leftMaxX = (1 << effectiveZoom) - 1 + val rightMinX = 0 + val rightMaxX = lngToTileX(bounds.right, zoom) val leftGroups = SQL""" SELECT id, z, x, y, group_type, centroid_lat, centroid_lng, @@ -277,11 +294,12 @@ class TileAggregateRepository @Inject() (override val db: Database) extends Repo def latToTileY(lat: Double, zoom: Int): Int = { val effectiveZoom = if (zoom < 14) zoom + ZOOM_OFFSET else zoom - val latClamped = math.max(-85.0511, math.min(85.0511, lat)) - val latRad = math.toRadians(latClamped) + val latClamped = math.max(-85.0511, math.min(85.0511, lat)) + val latRad = math.toRadians(latClamped) math .floor( - (1.0 - math.log(math.tan(latRad) + 1.0 / math.cos(latRad)) / math.Pi) / 2.0 * (1 << effectiveZoom) + (1.0 - math + .log(math.tan(latRad) + 1.0 / math.cos(latRad)) / math.Pi) / 2.0 * (1 << effectiveZoom) ) .toInt } @@ -293,7 +311,7 @@ class TileAggregateRepository @Inject() (override val db: Database) extends Repo def tileToLat(y: Int, zoom: Int): Double = { val effectiveZoom = if (zoom < 14) zoom + ZOOM_OFFSET else zoom - val n = math.Pi - 2.0 * math.Pi * y.toDouble / (1 << effectiveZoom) + val n = math.Pi - 2.0 * math.Pi * y.toDouble / (1 << effectiveZoom) math.toDegrees(math.atan(math.sinh(n))) } } diff --git a/app/org/maproulette/framework/service/NominatimService.scala b/app/org/maproulette/framework/service/NominatimService.scala index 227746f61..c3d9e4b57 100644 --- a/app/org/maproulette/framework/service/NominatimService.scala +++ b/app/org/maproulette/framework/service/NominatimService.scala @@ -81,19 +81,16 @@ class NominatimService @Inject() (wsClient: WSClient)(implicit ec: ExecutionCont // Convert the GeoJSON geometry to a WKT string for PostGIS convertGeoJSONToWKT(geometry) case None => - None } } else { - + None } } catch { case _: java.util.concurrent.TimeoutException => - None case _: Exception => - None } } @@ -118,11 +115,8 @@ class NominatimService @Inject() (wsClient: WSClient)(implicit ec: ExecutionCont case (Some("Point"), Some(coords)) if coords.value.size >= 2 => Some(pointToWKT(coords)) case (Some("LineString"), Some(coords)) if coords.value.size >= 2 => - - None case (Some("GeometryCollection"), _) => - None case _ => None From 837bce7358af6a6e362f6739e4f91d3ca7f26858 Mon Sep 17 00:00:00 2001 From: Collin Beczak Date: Tue, 17 Feb 2026 14:58:15 -0600 Subject: [PATCH 5/6] Add new endpoint to retrieve task data by tile coordinates - Implemented `getTaskTile` method in `TaskController` to fetch task data for specific tile coordinates (z/x/y) for zoom levels 14 and above. - Enhanced `TileAggregateRepository` with `getTaskGroupsByTile` method to retrieve pre-computed task groups for specified tile coordinates. - Updated `TileAggregateService` to include `getTileDataByCoords` method for processing tile data with optional difficulty and global filters. - Added corresponding API route for the new endpoint in `task.api` for improved frontend caching capabilities. --- .../framework/controller/TaskController.scala | 33 ++++++++++++++++ .../repository/TileAggregateRepository.scala | 20 ++++++++++ .../service/TileAggregateService.scala | 38 +++++++++++++++++++ conf/v2_route/task.api | 9 +++++ 4 files changed, 100 insertions(+) diff --git a/app/org/maproulette/framework/controller/TaskController.scala b/app/org/maproulette/framework/controller/TaskController.scala index 2907818a9..97041119c 100644 --- a/app/org/maproulette/framework/controller/TaskController.scala +++ b/app/org/maproulette/framework/controller/TaskController.scala @@ -393,6 +393,39 @@ class TaskController @Inject() ( } } + /** + * Get task data for a specific tile (z/x/y). + * Used for zoom 14+ where frontend requests individual tiles for caching. + * + * @param z Zoom level + * @param x Tile X coordinate + * @param y Tile Y coordinate + * @param global Include global challenges + * @param difficulty Optional difficulty filter (1=Easy, 2=Normal, 3=Expert) + * @return TaskMarkerResponse with tasks for this tile + */ + def getTaskTile( + z: Int, + x: Int, + y: Int, + global: Boolean, + difficulty: Option[Int] + ): Action[AnyContent] = Action.async { implicit request => + this.sessionManager.userAwareRequest { implicit user => + val validZoom = math.max(0, math.min(22, z)) + val validDifficulty = difficulty.filter(d => d >= 1 && d <= 3) + + val response = this.serviceManager.tileAggregate.getTileDataByCoords( + validZoom, + x, + y, + validDifficulty, + global + ) + Ok(Json.toJson(response)) + } + } + // for getting more detailed task marker data on individul makrers // def getTaskMarkerData(id: Long): Action[AnyContent] = Action.async { implicit request => // this.sessionManager.userAwareRequest { implicit user => diff --git a/app/org/maproulette/framework/repository/TileAggregateRepository.scala b/app/org/maproulette/framework/repository/TileAggregateRepository.scala index 7d653bad2..021fb6038 100644 --- a/app/org/maproulette/framework/repository/TileAggregateRepository.scala +++ b/app/org/maproulette/framework/repository/TileAggregateRepository.scala @@ -151,6 +151,26 @@ class TileAggregateRepository @Inject() (override val db: Database) extends Repo } } + /** + * Get pre-computed task groups for a specific tile (z, x, y). + * Used for zoom 14+ where frontend requests individual tiles for caching. + */ + def getTaskGroupsByTile( + z: Int, + x: Int, + y: Int + )(implicit c: Option[Connection] = None): List[TileTaskGroup] = { + this.withMRConnection { implicit c => + SQL""" + SELECT id, z, x, y, group_type, centroid_lat, centroid_lng, + task_ids, task_count, counts_by_filter::text as counts_by_filter + FROM tile_task_groups + WHERE z = $z AND x = $x AND y = $y + AND task_count > 0 + """.as(tileTaskGroupParser.*) + } + } + /** * Get pre-computed task groups within a polygon at a specific zoom level. * Filters groups whose centroids fall within the polygon. diff --git a/app/org/maproulette/framework/service/TileAggregateService.scala b/app/org/maproulette/framework/service/TileAggregateService.scala index 6783594ae..c1f1c06b3 100644 --- a/app/org/maproulette/framework/service/TileAggregateService.scala +++ b/app/org/maproulette/framework/service/TileAggregateService.scala @@ -222,6 +222,44 @@ class TileAggregateService @Inject() ( } } + /** + * Get tile data for a specific tile (z, x, y). + * Used for zoom 14+ where frontend requests individual tiles for caching. + */ + def getTileDataByCoords( + z: Int, + x: Int, + y: Int, + difficulty: Option[Int] = None, + global: Boolean = false + ): TaskMarkerResponse = { + // For zoom > 14, convert tile coords to zoom 14 and query that tile + // Zoom 16 has 4x the tiles per dimension as zoom 14, so divide by 2^(z-14) + val (queryZoom, queryX, queryY) = if (z > MAX_PRECOMPUTED_ZOOM) { + val zoomDiff = z - MAX_PRECOMPUTED_ZOOM + val scale = 1 << zoomDiff // 2^zoomDiff + (MAX_PRECOMPUTED_ZOOM, x / scale, y / scale) + } else { + (z, x, y) + } + + val taskGroups = repository.getTaskGroupsByTile(queryZoom, queryX, queryY) + + // Apply difficulty/global filters + val filteredGroups = taskGroups.flatMap { group => + val filteredCount = group.getFilteredCount(difficulty, global) + if (filteredCount > 0) Some((group, filteredCount)) else None + } + + val totalCount = filteredGroups.map(_._2).sum + + if (totalCount == 0) { + return TaskMarkerResponse(totalCount = 0) + } + + returnMixedResponse(filteredGroups, totalCount) + } + /** * Full rebuild of a specific zoom level */ diff --git a/conf/v2_route/task.api b/conf/v2_route/task.api index 7162f1469..b871fb510 100644 --- a/conf/v2_route/task.api +++ b/conf/v2_route/task.api @@ -742,6 +742,15 @@ GET /taskMarkers @org.maproulette.framework.controller.TaskController GET /taskTiles/:z @org.maproulette.framework.controller.TaskController.getTaskTiles(z: Int, bounds: String, global: Boolean ?= false, location_id: Option[Long] ?= None, keywords: Option[String] ?= None, difficulty: Option[Int] ?= None) ### # tags: [ Task ] +# summary: Get Task Tile Data by Coordinates +# description: Returns task data for a specific tile (z/x/y). Used for zoom 14+ where individual tiles can be cached. +# responses: +# '200': +# description: Task marker response for the tile +### +GET /taskTile/:z/:x/:y @org.maproulette.framework.controller.TaskController.getTaskTile(z: Int, x: Int, y: Int, global: Boolean ?= false, difficulty: Option[Int] ?= None) +### +# tags: [ Task ] # operationId: task_update_task_changeset # summary: Update Task Changeset # description: Will update the changeset of the task. It will do this by attempting to match the OSM changeset to the Task based on the geometry and the time that the changeset was executed. From 94badae02abcfc0b9e43436c74618a154f1c184c Mon Sep 17 00:00:00 2001 From: Collin Beczak Date: Tue, 17 Feb 2026 19:27:11 -0600 Subject: [PATCH 6/6] run formatting --- .../maproulette/framework/service/TileAggregateService.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/org/maproulette/framework/service/TileAggregateService.scala b/app/org/maproulette/framework/service/TileAggregateService.scala index c1f1c06b3..ccc6630cc 100644 --- a/app/org/maproulette/framework/service/TileAggregateService.scala +++ b/app/org/maproulette/framework/service/TileAggregateService.scala @@ -237,7 +237,7 @@ class TileAggregateService @Inject() ( // Zoom 16 has 4x the tiles per dimension as zoom 14, so divide by 2^(z-14) val (queryZoom, queryX, queryY) = if (z > MAX_PRECOMPUTED_ZOOM) { val zoomDiff = z - MAX_PRECOMPUTED_ZOOM - val scale = 1 << zoomDiff // 2^zoomDiff + val scale = 1 << zoomDiff // 2^zoomDiff (MAX_PRECOMPUTED_ZOOM, x / scale, y / scale) } else { (z, x, y)