diff --git a/app/org/maproulette/Config.scala b/app/org/maproulette/Config.scala index 47fb8d769..ca441bc1d 100644 --- a/app/org/maproulette/Config.scala +++ b/app/org/maproulette/Config.scala @@ -348,6 +348,7 @@ object Config { val KEY_SCHEDULER_SNAPSHOT_CHALLENGES_INTERVAL = s"$SUB_GROUP_SCHEDULER.challengesSnapshot.interval" val KEY_SCHEDULER_SNAPSHOT_CHALLENGES_START = s"$SUB_GROUP_SCHEDULER.challengesSnapshot.startTime" + val KEY_SCHEDULER_TILE_REFRESH_INTERVAL = s"$SUB_GROUP_SCHEDULER.tileRefresh.interval" val KEY_MAPROULETTE_FRONTEND = s"$GROUP_MAPROULETTE.frontend" val SUB_GROUP_MAPILLARY = s"$GROUP_MAPROULETTE.mapillary" diff --git a/app/org/maproulette/framework/controller/TaskController.scala b/app/org/maproulette/framework/controller/TaskController.scala index d3f2397fb..97041119c 100644 --- a/app/org/maproulette/framework/controller/TaskController.scala +++ b/app/org/maproulette/framework/controller/TaskController.scala @@ -327,6 +327,105 @@ class TaskController @Inject() ( } } + /** + * Gets task data using pre-computed tile aggregates for efficient map display at scale. + * Uses a tile pyramid system with pre-computed counts broken down by difficulty × global. + * + * Behavior by filter: + * - difficulty & global: Filtered from pre-computed tile data (fast) + * - location_id: Recursive tile drilling until within polygon or < 2000 tasks + * - keywords: Falls back to dynamic query (challenge-level filter, not pre-computed) + * + * All fetched data is re-clustered into ~80 clusters for display. + * When total tasks < 2000, returns individual task markers instead of clusters. + * + * @param z Zoom level (0-14 for pre-computed tiles) + * @param bounds Comma-separated bounding box: left,bottom,right,top + * @param global Whether to include global challenges + * @param location_id Optional Nominatim place_id for polygon filtering + * @param keywords Optional keywords filter (triggers fallback to dynamic query) + * @param difficulty Optional difficulty filter (1=Easy, 2=Normal, 3=Expert) + * @return TaskMarkerResponse with totalCount and either clusters or tasks (with overlaps) + */ + def getTaskTiles( + z: Int, + bounds: String, + global: Boolean, + location_id: Option[Long], + keywords: Option[String], + difficulty: Option[Int] + ): Action[AnyContent] = Action.async { implicit request => + this.sessionManager.userAwareRequest { implicit user => + // Clamp zoom to valid range + val validZoom = math.max(0, math.min(20, z)) + + // Only allow valid difficulty values (1, 2, 3) + val validDifficulty = difficulty.filter(d => d >= 1 && d <= 3) + + // Parse and validate bounding box + val boundingBox = + try { + bounds.split(",").map(_.trim.toDouble).toList match { + case List(left, bottom, right, top) => + // Clamp coordinates to valid ranges + val clampedLeft = math.max(-180.0, math.min(180.0, left)) + val clampedRight = math.max(-180.0, math.min(180.0, right)) + val clampedBottom = math.max(-90.0, math.min(90.0, bottom)) + val clampedTop = math.max(-90.0, math.min(90.0, top)) + SearchLocation(clampedLeft, clampedBottom, clampedRight, clampedTop) + case _ => + SearchLocation(-180.0, -90.0, 180.0, 90.0) + } + } catch { + case _: NumberFormatException => + SearchLocation(-180.0, -90.0, 180.0, 90.0) + } + + val response = this.serviceManager.tileAggregate.getTileData( + validZoom, + boundingBox, + validDifficulty, + global, + location_id, + keywords + ) + Ok(Json.toJson(response)) + } + } + + /** + * Get task data for a specific tile (z/x/y). + * Used for zoom 14+ where frontend requests individual tiles for caching. + * + * @param z Zoom level + * @param x Tile X coordinate + * @param y Tile Y coordinate + * @param global Include global challenges + * @param difficulty Optional difficulty filter (1=Easy, 2=Normal, 3=Expert) + * @return TaskMarkerResponse with tasks for this tile + */ + def getTaskTile( + z: Int, + x: Int, + y: Int, + global: Boolean, + difficulty: Option[Int] + ): Action[AnyContent] = Action.async { implicit request => + this.sessionManager.userAwareRequest { implicit user => + val validZoom = math.max(0, math.min(22, z)) + val validDifficulty = difficulty.filter(d => d >= 1 && d <= 3) + + val response = this.serviceManager.tileAggregate.getTileDataByCoords( + validZoom, + x, + y, + validDifficulty, + global + ) + Ok(Json.toJson(response)) + } + } + // for getting more detailed task marker data on individul makrers // def getTaskMarkerData(id: Long): Action[AnyContent] = Action.async { implicit request => // this.sessionManager.userAwareRequest { implicit user => diff --git a/app/org/maproulette/framework/model/TileAggregate.scala b/app/org/maproulette/framework/model/TileAggregate.scala new file mode 100644 index 000000000..328e26a11 --- /dev/null +++ b/app/org/maproulette/framework/model/TileAggregate.scala @@ -0,0 +1,135 @@ +/* + * Copyright (C) 2020 MapRoulette contributors (see CONTRIBUTORS.md). + * Licensed under the Apache License, Version 2.0 (see LICENSE). + */ +package org.maproulette.framework.model +import play.api.libs.json._ + +case class ClusterPoint( + lat: Double, + lng: Double, + count: Int +) + +object ClusterPoint { + implicit val clusterPointFormat: Format[ClusterPoint] = Json.format[ClusterPoint] +} + +case class FilterCounts( + d1_gf: Int = 0, + d1_gt: Int = 0, + d2_gf: Int = 0, + d2_gt: Int = 0, + d3_gf: Int = 0, + d3_gt: Int = 0, + d0_gf: Int = 0, + d0_gt: Int = 0 +) { + + /** + * Get count for specific difficulty and global filters + * + * @param difficulty Optional difficulty filter (1, 2, 3) + * @param global Whether to include global challenges (true = all, false = non-global only) + * @return Filtered count + */ + def getFilteredCount(difficulty: Option[Int], global: Boolean): Int = { + difficulty match { + case Some(1) => if (global) d1_gf + d1_gt else d1_gf + case Some(2) => if (global) d2_gf + d2_gt else d2_gf + case Some(3) => if (global) d3_gf + d3_gt else d3_gf + case Some(_) => + // Unknown difficulty values use d0 (unset/other) bucket + if (global) d0_gf + d0_gt else d0_gf + case None => + // No filter: sum all difficulty levels + if (global) d1_gf + d1_gt + d2_gf + d2_gt + d3_gf + d3_gt + d0_gf + d0_gt + else d1_gf + d2_gf + d3_gf + d0_gf + } + } + + /** + * Get total count (all combinations) + */ + def total: Int = d1_gf + d1_gt + d2_gf + d2_gt + d3_gf + d3_gt + d0_gf + d0_gt +} + +object FilterCounts { + implicit val filterCountsFormat: Format[FilterCounts] = Json.format[FilterCounts] + + def fromJson(json: JsValue): FilterCounts = { + FilterCounts( + d1_gf = (json \ "d1_gf").asOpt[Int].getOrElse(0), + d1_gt = (json \ "d1_gt").asOpt[Int].getOrElse(0), + d2_gf = (json \ "d2_gf").asOpt[Int].getOrElse(0), + d2_gt = (json \ "d2_gt").asOpt[Int].getOrElse(0), + d3_gf = (json \ "d3_gf").asOpt[Int].getOrElse(0), + d3_gt = (json \ "d3_gt").asOpt[Int].getOrElse(0), + d0_gf = (json \ "d0_gf").asOpt[Int].getOrElse(0), + d0_gt = (json \ "d0_gt").asOpt[Int].getOrElse(0) + ) + } +} + +/** + * Represents a pre-computed task group at any zoom level (0-14). + * + * Zoom 0-13: One cluster per tile (group_type=2, no task_ids) + * - As zoom increases, tiles get smaller, clusters naturally split + * - Frontend displays these as cluster markers + * + * Zoom 14: One entry per overlap group (group_type=0 or 1, with task_ids) + * - Frontend handles clustering for zoom levels 14-22 + * - Returns individual task markers and overlapping task markers + * + * @param id Database ID + * @param z Zoom level (0-14) + * @param x Tile X coordinate + * @param y Tile Y coordinate + * @param groupType 0=single task, 1=overlapping tasks, 2=cluster + * @param centroidLat Centroid latitude of the group + * @param centroidLng Centroid longitude of the group + * @param taskIds List of task IDs (empty for clusters at zoom 0-13) + * @param taskCount Number of tasks in this group + * @param countsByFilter Counts broken down by difficulty × global for filtering + */ +case class TileTaskGroup( + id: Long, + z: Int, + x: Int, + y: Int, + groupType: Int, + centroidLat: Double, + centroidLng: Double, + taskIds: List[Long], + taskCount: Int, + countsByFilter: FilterCounts +) { + + /** + * Get the filtered count for this group based on difficulty and global filters + */ + def getFilteredCount(difficulty: Option[Int], global: Boolean): Int = { + countsByFilter.getFilteredCount(difficulty, global) + } + + /** + * Check if this is a single task (zoom 14 only) + */ + def isSingle: Boolean = groupType == 0 + + /** + * Check if this is an overlapping group (zoom 14 only) + */ + def isOverlapping: Boolean = groupType == 1 + + /** + * Check if this is a cluster (zoom 0-13) + */ + def isCluster: Boolean = groupType == 2 +} + +object TileTaskGroup { + implicit val tileTaskGroupWrites: Writes[TileTaskGroup] = Json.writes[TileTaskGroup] + implicit val tileTaskGroupReads: Reads[TileTaskGroup] = Json.reads[TileTaskGroup] +} diff --git a/app/org/maproulette/framework/repository/TileAggregateRepository.scala b/app/org/maproulette/framework/repository/TileAggregateRepository.scala new file mode 100644 index 000000000..021fb6038 --- /dev/null +++ b/app/org/maproulette/framework/repository/TileAggregateRepository.scala @@ -0,0 +1,337 @@ +/* + * Copyright (C) 2020 MapRoulette contributors (see CONTRIBUTORS.md). + * Licensed under the Apache License, Version 2.0 (see LICENSE). + */ + +package org.maproulette.framework.repository + +import java.sql.Connection + +import anorm._ +import anorm.SqlParser.{get, int, double} +import anorm.postgresql._ +import javax.inject.{Inject, Singleton} +import org.maproulette.framework.model.{ + TileTaskGroup, + FilterCounts, + TaskMarker, + TaskMarkerLocation, + ClusterPoint +} +import org.maproulette.session.SearchLocation +import play.api.db.Database +import play.api.libs.json.Json + +/** + * Repository for accessing pre-computed tile task groups. + * All zoom levels (0-14) use overlap detection via ST_ClusterDBSCAN. + * Groups are categorized as single (group_type=0) or overlapping (group_type=1). + */ +@Singleton +class TileAggregateRepository @Inject() (override val db: Database) extends RepositoryMixin { + implicit val baseTable: String = "tile_task_groups" + + // Simplification tolerance in degrees (~1km at equator) + private val SIMPLIFY_TOLERANCE = 0.01 + + private val tileTaskGroupParser: RowParser[TileTaskGroup] = { + get[Long]("id") ~ + get[Int]("z") ~ + get[Int]("x") ~ + get[Int]("y") ~ + get[Int]("group_type") ~ + get[Double]("centroid_lat") ~ + get[Double]("centroid_lng") ~ + get[List[Long]]("task_ids") ~ + get[Int]("task_count") ~ + get[Option[String]]("counts_by_filter") map { + case id ~ z ~ x ~ y ~ groupType ~ centroidLat ~ centroidLng ~ taskIds ~ taskCount ~ countsJson => + val filterCounts = countsJson + .map { json => + try { + FilterCounts.fromJson(Json.parse(json)) + } catch { + case _: Exception => FilterCounts() + } + } + .getOrElse(FilterCounts()) + + TileTaskGroup( + id, + z, + x, + y, + groupType, + centroidLat, + centroidLng, + taskIds, + taskCount, + filterCounts + ) + } + } + + private val clusterPointParser: RowParser[ClusterPoint] = { + get[Double]("lat") ~ + get[Double]("lng") ~ + get[Int]("count") map { + case lat ~ lng ~ count => + ClusterPoint(lat, lng, count) + } + } + + private val taskMarkerParser: RowParser[TaskMarker] = { + get[Long]("id") ~ + get[Double]("lat") ~ + get[Double]("lng") ~ + get[Int]("status") ~ + get[Int]("priority") ~ + get[Option[Long]]("bundle_id") ~ + get[Option[Long]]("locked_by") map { + case id ~ lat ~ lng ~ status ~ priority ~ bundleId ~ lockedBy => + TaskMarker(id, TaskMarkerLocation(lat, lng), status, priority, bundleId, lockedBy) + } + } + + /** + * Get pre-computed task groups in a bounding box at a specific zoom level. + * Handles anti-meridian crossing (when bounds.left > bounds.right). + */ + def getTaskGroupsInBounds( + zoom: Int, + bounds: SearchLocation + )(implicit c: Option[Connection] = None): List[TileTaskGroup] = { + this.withMRConnection { implicit c => + val minY = latToTileY(bounds.top, zoom) + val maxY = latToTileY(bounds.bottom, zoom) + + if (bounds.left > bounds.right) { + // Anti-meridian crossing + val effectiveZoom = if (zoom < 14) zoom + ZOOM_OFFSET else zoom + val leftMinX = lngToTileX(bounds.left, zoom) + val leftMaxX = (1 << effectiveZoom) - 1 + val rightMinX = 0 + val rightMaxX = lngToTileX(bounds.right, zoom) + + val leftGroups = SQL""" + SELECT id, z, x, y, group_type, centroid_lat, centroid_lng, + task_ids, task_count, counts_by_filter::text as counts_by_filter + FROM tile_task_groups + WHERE z = $zoom + AND x >= $leftMinX AND x <= $leftMaxX + AND y >= $minY AND y <= $maxY + AND task_count > 0 + """.as(tileTaskGroupParser.*) + + val rightGroups = SQL""" + SELECT id, z, x, y, group_type, centroid_lat, centroid_lng, + task_ids, task_count, counts_by_filter::text as counts_by_filter + FROM tile_task_groups + WHERE z = $zoom + AND x >= $rightMinX AND x <= $rightMaxX + AND y >= $minY AND y <= $maxY + AND task_count > 0 + """.as(tileTaskGroupParser.*) + + leftGroups ++ rightGroups + } else { + val minX = lngToTileX(bounds.left, zoom) + val maxX = lngToTileX(bounds.right, zoom) + + SQL""" + SELECT id, z, x, y, group_type, centroid_lat, centroid_lng, + task_ids, task_count, counts_by_filter::text as counts_by_filter + FROM tile_task_groups + WHERE z = $zoom + AND x >= $minX AND x <= $maxX + AND y >= $minY AND y <= $maxY + AND task_count > 0 + """.as(tileTaskGroupParser.*) + } + } + } + + /** + * Get pre-computed task groups for a specific tile (z, x, y). + * Used for zoom 14+ where frontend requests individual tiles for caching. + */ + def getTaskGroupsByTile( + z: Int, + x: Int, + y: Int + )(implicit c: Option[Connection] = None): List[TileTaskGroup] = { + this.withMRConnection { implicit c => + SQL""" + SELECT id, z, x, y, group_type, centroid_lat, centroid_lng, + task_ids, task_count, counts_by_filter::text as counts_by_filter + FROM tile_task_groups + WHERE z = $z AND x = $x AND y = $y + AND task_count > 0 + """.as(tileTaskGroupParser.*) + } + } + + /** + * Get pre-computed task groups within a polygon at a specific zoom level. + * Filters groups whose centroids fall within the polygon. + */ + def getTaskGroupsInPolygon( + zoom: Int, + polygonWkt: String, + bounds: SearchLocation + )(implicit c: Option[Connection] = None): List[TileTaskGroup] = { + this.withMRConnection { implicit c => + val minX = lngToTileX(bounds.left, zoom) + val maxX = lngToTileX(bounds.right, zoom) + val minY = latToTileY(bounds.top, zoom) + val maxY = latToTileY(bounds.bottom, zoom) + + SQL""" + WITH simplified AS ( + SELECT ST_Simplify(ST_GeomFromText($polygonWkt, 4326), $SIMPLIFY_TOLERANCE) as geom + ) + SELECT id, z, x, y, group_type, centroid_lat, centroid_lng, + task_ids, task_count, counts_by_filter::text as counts_by_filter + FROM tile_task_groups + CROSS JOIN simplified + WHERE z = $zoom + AND x >= $minX AND x <= $maxX + AND y >= $minY AND y <= $maxY + AND task_count > 0 + AND ST_Contains(simplified.geom, ST_SetSRID(ST_MakePoint(centroid_lng, centroid_lat), 4326)) + """.as(tileTaskGroupParser.*) + } + } + + /** + * Fetch all task markers in a bounding box with a single query. + * Used when total count is low enough to return individual tasks. + */ + def getTaskMarkersInBounds( + bounds: SearchLocation, + difficulty: Option[Int] = None, + global: Boolean = false, + limit: Int = 2000 + )(implicit c: Option[Connection] = None): List[TaskMarker] = { + this.withMRConnection { implicit c => + val left = bounds.left + val bottom = bounds.bottom + val right = bounds.right + val top = bounds.top + + // SQL SAFETY: These use #$ string interpolation which is safe here because: + // - globalFilter is a hardcoded string literal (no user input) + // - difficultyFilter uses a validated Int from difficulty.map, not user-provided strings + val globalFilter = if (!global) "AND c.is_global = false" else "" + val difficultyFilter = difficulty.map(d => s"AND c.difficulty = $d").getOrElse("") + + SQL""" + SELECT DISTINCT tasks.id, ST_Y(tasks.location) as lat, ST_X(tasks.location) as lng, + tasks.status, tasks.priority, tasks.bundle_id, l.user_id as locked_by + FROM tasks + INNER JOIN challenges c ON c.id = tasks.parent_id + INNER JOIN projects p ON p.id = c.parent_id + LEFT JOIN locked l ON l.item_id = tasks.id AND l.item_type = 2 + WHERE tasks.location && ST_MakeEnvelope($left, $bottom, $right, $top, 4326) + AND ST_Intersects(tasks.location, ST_MakeEnvelope($left, $bottom, $right, $top, 4326)) + AND tasks.status IN (0, 3, 6) + AND c.deleted = false AND c.enabled = true AND c.is_archived = false + AND p.deleted = false AND p.enabled = true + #$globalFilter + #$difficultyFilter + LIMIT $limit + """.as(taskMarkerParser.*) + } + } + + /** + * Fetch task markers within a simplified polygon. + */ + def getTaskMarkersInPolygonSimplified( + polygonWkt: String, + difficulty: Option[Int] = None, + global: Boolean = false, + limit: Option[Int] = None + )(implicit c: Option[Connection] = None): List[TaskMarker] = { + this.withMRConnection { implicit c => + val globalFilter = if (!global) "AND c.is_global = false" else "" + val difficultyFilter = difficulty.map(d => s"AND c.difficulty = $d").getOrElse("") + val limitClause = limit.map(l => s"LIMIT $l").getOrElse("") + + SQL""" + WITH simplified AS ( + SELECT ST_Simplify(ST_GeomFromText($polygonWkt, 4326), $SIMPLIFY_TOLERANCE) as geom + ) + SELECT DISTINCT tasks.id, ST_Y(tasks.location) as lat, ST_X(tasks.location) as lng, + tasks.status, tasks.priority, tasks.bundle_id, l.user_id as locked_by + FROM tasks + CROSS JOIN simplified + INNER JOIN challenges c ON c.id = tasks.parent_id + INNER JOIN projects p ON p.id = c.parent_id + LEFT JOIN locked l ON l.item_id = tasks.id AND l.item_type = 2 + WHERE tasks.location && simplified.geom + AND ST_Intersects(tasks.location, simplified.geom) + AND tasks.status IN (0, 3, 6) + AND c.deleted = false AND c.enabled = true AND c.is_archived = false + AND p.deleted = false AND p.enabled = true + #$globalFilter + #$difficultyFilter + #$limitClause + """.as(taskMarkerParser.*) + } + } + + /** + * Rebuild a specific zoom level with overlap detection + */ + def rebuildZoomLevel(zoom: Int)(implicit c: Option[Connection] = None): Int = { + this.withMRTransaction { implicit c => + SQL"SELECT rebuild_zoom_level($zoom)" + .as(SqlParser.int("rebuild_zoom_level").single) + } + } + + /** + * Get total count of pre-computed task groups + */ + def getTotalTaskGroupCount()(implicit c: Option[Connection] = None): Int = { + this.withMRConnection { implicit c => + SQL"SELECT COUNT(*)::int as count FROM tile_task_groups" + .as(SqlParser.int("count").single) + } + } + + // Offset for tile coordinate calculations (zoom 0 uses zoom 2 grid, etc.) + // Must match the offset in rebuild_zoom_level() SQL function + private val ZOOM_OFFSET = 2 + + // Web Mercator coordinate conversion functions + // For zoom 0-13, we use effectiveZoom = zoom + ZOOM_OFFSET for more granular tiles + def lngToTileX(lng: Double, zoom: Int): Int = { + val effectiveZoom = if (zoom < 14) zoom + ZOOM_OFFSET else zoom + math.floor((lng + 180.0) / 360.0 * (1 << effectiveZoom)).toInt + } + + def latToTileY(lat: Double, zoom: Int): Int = { + val effectiveZoom = if (zoom < 14) zoom + ZOOM_OFFSET else zoom + val latClamped = math.max(-85.0511, math.min(85.0511, lat)) + val latRad = math.toRadians(latClamped) + math + .floor( + (1.0 - math + .log(math.tan(latRad) + 1.0 / math.cos(latRad)) / math.Pi) / 2.0 * (1 << effectiveZoom) + ) + .toInt + } + + def tileToLng(x: Int, zoom: Int): Double = { + val effectiveZoom = if (zoom < 14) zoom + ZOOM_OFFSET else zoom + x.toDouble / (1 << effectiveZoom) * 360.0 - 180.0 + } + + def tileToLat(y: Int, zoom: Int): Double = { + val effectiveZoom = if (zoom < 14) zoom + ZOOM_OFFSET else zoom + val n = math.Pi - 2.0 * math.Pi * y.toDouble / (1 << effectiveZoom) + math.toDegrees(math.atan(math.sinh(n))) + } +} diff --git a/app/org/maproulette/framework/service/NominatimService.scala b/app/org/maproulette/framework/service/NominatimService.scala index 8c9aac969..c3d9e4b57 100644 --- a/app/org/maproulette/framework/service/NominatimService.scala +++ b/app/org/maproulette/framework/service/NominatimService.scala @@ -47,6 +47,7 @@ class NominatimService @Inject() (wsClient: WSClient)(implicit ec: ExecutionCont /** * Fetches polygon geometry from Nominatim API (not cached) + * Note: Uses blocking call - consider using async version for high-throughput scenarios. * * @param placeId The Nominatim place_id * @return Option containing the WKT polygon string @@ -57,6 +58,7 @@ class NominatimService @Inject() (wsClient: WSClient)(implicit ec: ExecutionCont val futureResponse = wsClient .url(url) + .withRequestTimeout(REQUEST_TIMEOUT) .addQueryStringParameters( "place_id" -> placeId.toString, "format" -> "json", @@ -68,7 +70,7 @@ class NominatimService @Inject() (wsClient: WSClient)(implicit ec: ExecutionCont ) .get() - val response = Await.result(futureResponse, REQUEST_TIMEOUT) + val response = Await.result(futureResponse, REQUEST_TIMEOUT + 1.second) if (response.status == 200) { val json = response.json @@ -82,32 +84,45 @@ class NominatimService @Inject() (wsClient: WSClient)(implicit ec: ExecutionCont None } } else { + None } } catch { - case e: Exception => + case _: java.util.concurrent.TimeoutException => + None + case _: Exception => None } } /** - * Converts a GeoJSON geometry object to WKT (Well-Known Text) format for PostGIS + * Converts a GeoJSON geometry object to WKT (Well-Known Text) format for PostGIS. + * Handles Polygon, MultiPolygon, Point, and LineString geometry types. * * @param geometry The GeoJSON geometry object - * @return Option containing the WKT string, or None if conversion fails + * @return Option containing the WKT string, or None if conversion fails or type unsupported */ private def convertGeoJSONToWKT(geometry: JsObject): Option[String] = { - val geometryType = (geometry \ "type").asOpt[String] - val coordinates = (geometry \ "coordinates").asOpt[JsArray] - - (geometryType, coordinates) match { - case (Some("Polygon"), Some(coords)) => - Some(polygonToWKT(coords)) - case (Some("MultiPolygon"), Some(coords)) => - Some(multiPolygonToWKT(coords)) - case (Some("Point"), Some(coords)) => - Some(pointToWKT(coords)) - case _ => + try { + val geometryType = (geometry \ "type").asOpt[String] + val coordinates = (geometry \ "coordinates").asOpt[JsArray] + + (geometryType, coordinates) match { + case (Some("Polygon"), Some(coords)) if coords.value.nonEmpty => + Some(polygonToWKT(coords)) + case (Some("MultiPolygon"), Some(coords)) if coords.value.nonEmpty => + Some(multiPolygonToWKT(coords)) + case (Some("Point"), Some(coords)) if coords.value.size >= 2 => + Some(pointToWKT(coords)) + case (Some("LineString"), Some(coords)) if coords.value.size >= 2 => + None + case (Some("GeometryCollection"), _) => + None + case _ => + None + } + } catch { + case _: Exception => None } } diff --git a/app/org/maproulette/framework/service/ServiceManager.scala b/app/org/maproulette/framework/service/ServiceManager.scala index 55ad2fab8..291fc7b35 100644 --- a/app/org/maproulette/framework/service/ServiceManager.scala +++ b/app/org/maproulette/framework/service/ServiceManager.scala @@ -39,7 +39,8 @@ class ServiceManager @Inject() ( notificationService: Provider[NotificationService], leaderboardService: Provider[LeaderboardService], taskHistoryService: Provider[TaskHistoryService], - nominatimService: Provider[NominatimService] + nominatimService: Provider[NominatimService], + tileAggregateService: Provider[TileAggregateService] ) { def comment: CommentService = commentService.get() @@ -98,4 +99,6 @@ class ServiceManager @Inject() ( def leaderboard: LeaderboardService = leaderboardService.get() def nominatim: NominatimService = nominatimService.get() + + def tileAggregate: TileAggregateService = tileAggregateService.get() } diff --git a/app/org/maproulette/framework/service/TileAggregateService.scala b/app/org/maproulette/framework/service/TileAggregateService.scala new file mode 100644 index 000000000..ccc6630cc --- /dev/null +++ b/app/org/maproulette/framework/service/TileAggregateService.scala @@ -0,0 +1,292 @@ +/* + * Copyright (C) 2020 MapRoulette contributors (see CONTRIBUTORS.md). + * Licensed under the Apache License, Version 2.0 (see LICENSE). + */ + +package org.maproulette.framework.service + +import javax.inject.{Inject, Singleton} +import org.maproulette.framework.model.{ + TaskMarker, + TaskMarkerLocation, + TileTaskGroup, + ClusterPoint, + TaskMarkerResponse, + TaskClusterSummary, + OverlappingTaskMarker, + Point +} +import org.maproulette.framework.repository.{TileAggregateRepository, TaskClusterRepository} +import org.maproulette.session.SearchLocation +import org.slf4j.LoggerFactory +import play.api.libs.json.Json + +import scala.collection.mutable.ListBuffer + +/** + * Service layer for tile-based task aggregation. + * + * Zoom 0-13: Returns mix of singles, overlapping markers, and clusters + * - Tiles with 1 task at 1 location → single marker + * - Tiles with N tasks at 1 location → overlapping marker + * - Tiles with tasks at multiple locations → cluster + * Zoom 14+: Returns individual tasks + overlapping markers (frontend handles clustering) + */ +@Singleton +class TileAggregateService @Inject() ( + repository: TileAggregateRepository, + taskClusterRepository: TaskClusterRepository, + nominatimService: NominatimService +) { + private val logger = LoggerFactory.getLogger(this.getClass) + + // Maximum pre-computed zoom level (zoom 14+ all use zoom 14 data) + val MAX_PRECOMPUTED_ZOOM = 14 + + /** + * Get tile data for a bounding box with filtering. + * + * @param zoom Map zoom level (0-22) + * @param bounds Bounding box + * @param difficulty Optional difficulty filter + * @param global Include global challenges + * @param locationId Optional Nominatim place_id for polygon filtering + * @param keywords Optional keywords (triggers fallback to dynamic query) + * @return TaskMarkerResponse with clusters or tasks (including overlaps) + */ + def getTileData( + zoom: Int, + bounds: SearchLocation, + difficulty: Option[Int] = None, + global: Boolean = false, + locationId: Option[Long] = None, + keywords: Option[String] = None + ): TaskMarkerResponse = { + + // Keywords filter requires fallback (challenge-level filter, not pre-computed) + if (keywords.exists(_.trim.nonEmpty)) { + return getFallbackData(bounds, difficulty, global, locationId, keywords) + } + + // Determine which pre-computed zoom level to query + // Zoom 14-22 all use zoom 14 data (frontend clusters) + // Zoom 0-13 use their respective pre-computed clusters + val queryZoom = math.min(zoom, MAX_PRECOMPUTED_ZOOM) + + // Get polygon for location filtering if needed + val polygonWkt = locationId.flatMap(id => nominatimService.getPolygonByPlaceId(id)) + + // Fetch pre-computed groups + val taskGroups = polygonWkt match { + case Some(wkt) => repository.getTaskGroupsInPolygon(queryZoom, wkt, bounds) + case None => repository.getTaskGroupsInBounds(queryZoom, bounds) + } + + // Apply difficulty/global filters + val filteredGroups = taskGroups.flatMap { group => + val filteredCount = group.getFilteredCount(difficulty, global) + if (filteredCount > 0) Some((group, filteredCount)) else None + } + + val totalCount = filteredGroups.map(_._2).sum + + if (totalCount == 0) { + return TaskMarkerResponse(totalCount = 0) + } + + // Process all groups - separate by type + returnMixedResponse(filteredGroups, totalCount) + } + + /** + * Return mixed response with clusters, singles, and overlapping markers. + * Works for all zoom levels. + * + * Zoom 0-13: Mostly clusters, but isolated singles/overlaps are included + * Zoom 14+: Only singles and overlapping markers (no clusters) + */ + private def returnMixedResponse( + groups: List[(TileTaskGroup, Int)], + totalCount: Int + ): TaskMarkerResponse = { + val singleMarkers = ListBuffer[TaskMarker]() + val overlappingMarkers = ListBuffer[OverlappingTaskMarker]() + val clusterPoints = ListBuffer[(Point, Int)]() + + groups.foreach { + case (group, filteredCount) => + if (group.isSingle) { + // Single task - add as individual marker + group.taskIds.headOption.foreach { taskId => + singleMarkers += TaskMarker( + id = taskId, + location = TaskMarkerLocation(group.centroidLat, group.centroidLng), + status = 0, + priority = 0, + bundleId = None, + lockedBy = None + ) + } + } else if (group.isOverlapping) { + // Overlapping tasks - create overlapping marker with all tasks + val tasks = group.taskIds.map { taskId => + TaskMarker( + id = taskId, + location = TaskMarkerLocation(group.centroidLat, group.centroidLng), + status = 0, + priority = 0, + bundleId = None, + lockedBy = None + ) + } + overlappingMarkers += OverlappingTaskMarker( + TaskMarkerLocation(group.centroidLat, group.centroidLng), + tasks + ) + } else if (group.isCluster) { + // Cluster - add as cluster point + clusterPoints += ((Point(group.centroidLat, group.centroidLng), filteredCount)) + } + } + + // Build cluster summaries + val clusterSummaries = if (clusterPoints.nonEmpty) { + Some(clusterPoints.toList.zipWithIndex.map { + case ((point, count), idx) => + TaskClusterSummary( + clusterId = idx, + numberOfPoints = count, + taskId = None, + taskStatus = None, + point = point, + bounding = Json.obj() + ) + }) + } else { + None + } + + TaskMarkerResponse( + totalCount = totalCount, + tasks = if (singleMarkers.nonEmpty) Some(singleMarkers.toList) else None, + overlappingTasks = if (overlappingMarkers.nonEmpty) Some(overlappingMarkers.toList) else None, + clusters = clusterSummaries + ) + } + + /** + * Fallback to dynamic query for keywords filtering. + */ + private def getFallbackData( + bounds: SearchLocation, + difficulty: Option[Int], + global: Boolean, + locationId: Option[Long], + keywords: Option[String] + ): TaskMarkerResponse = { + val statusList = List(0, 3, 6) + + val taskCount = taskClusterRepository.queryCountTaskMarkers( + statusList, + global, + bounds, + locationId, + keywords, + difficulty + ) + + if (taskCount >= 2000) { + val clusters = taskClusterRepository.queryTaskMarkersClustered( + statusList, + global, + bounds, + locationId, + keywords, + difficulty + ) + TaskMarkerResponse(totalCount = taskCount, clusters = Some(clusters)) + } else { + val (singleMarkers, overlappingMarkers) = taskClusterRepository.queryTaskMarkersWithOverlaps( + statusList, + global, + bounds, + locationId, + keywords, + difficulty + ) + TaskMarkerResponse( + totalCount = taskCount, + tasks = Some(singleMarkers), + overlappingTasks = if (overlappingMarkers.nonEmpty) Some(overlappingMarkers) else None + ) + } + } + + /** + * Get tile data for a specific tile (z, x, y). + * Used for zoom 14+ where frontend requests individual tiles for caching. + */ + def getTileDataByCoords( + z: Int, + x: Int, + y: Int, + difficulty: Option[Int] = None, + global: Boolean = false + ): TaskMarkerResponse = { + // For zoom > 14, convert tile coords to zoom 14 and query that tile + // Zoom 16 has 4x the tiles per dimension as zoom 14, so divide by 2^(z-14) + val (queryZoom, queryX, queryY) = if (z > MAX_PRECOMPUTED_ZOOM) { + val zoomDiff = z - MAX_PRECOMPUTED_ZOOM + val scale = 1 << zoomDiff // 2^zoomDiff + (MAX_PRECOMPUTED_ZOOM, x / scale, y / scale) + } else { + (z, x, y) + } + + val taskGroups = repository.getTaskGroupsByTile(queryZoom, queryX, queryY) + + // Apply difficulty/global filters + val filteredGroups = taskGroups.flatMap { group => + val filteredCount = group.getFilteredCount(difficulty, global) + if (filteredCount > 0) Some((group, filteredCount)) else None + } + + val totalCount = filteredGroups.map(_._2).sum + + if (totalCount == 0) { + return TaskMarkerResponse(totalCount = 0) + } + + returnMixedResponse(filteredGroups, totalCount) + } + + /** + * Full rebuild of a specific zoom level + */ + def rebuildZoomLevel(zoom: Int): Int = { + logger.info(s"Starting rebuild of zoom level $zoom") + val groupsCreated = repository.rebuildZoomLevel(zoom) + logger.info(s"Completed rebuild of zoom level $zoom: $groupsCreated groups created") + groupsCreated + } + + /** + * Full rebuild of all zoom levels (0-14) + */ + def rebuildAllTiles(): Int = { + logger.info("Starting full rebuild of all tile task groups") + var totalGroups = 0 + for (zoom <- 0 to MAX_PRECOMPUTED_ZOOM) { + totalGroups += rebuildZoomLevel(zoom) + } + logger.info(s"Completed full rebuild: $totalGroups total groups created") + totalGroups + } + + /** + * Get statistics about the tile system + */ + def getStats(): Map[String, Int] = { + Map("totalTaskGroups" -> repository.getTotalTaskGroupCount()) + } +} diff --git a/app/org/maproulette/jobs/Scheduler.scala b/app/org/maproulette/jobs/Scheduler.scala index b2e324153..aa3a2fae2 100644 --- a/app/org/maproulette/jobs/Scheduler.scala +++ b/app/org/maproulette/jobs/Scheduler.scala @@ -115,6 +115,13 @@ class Scheduler @Inject() ( Config.KEY_SCHEDULER_UPDATE_CHALLENGE_COMPLETION_INTERVAL ) + schedule( + "refreshTileAggregates", + "Rebuilding tile aggregates", + 1.minute, + Config.KEY_SCHEDULER_TILE_REFRESH_INTERVAL + ) + scheduleAtTime( "sendCountNotificationDailyEmails", "Sending Count Notification Daily Emails", diff --git a/app/org/maproulette/jobs/SchedulerActor.scala b/app/org/maproulette/jobs/SchedulerActor.scala index 28ff6eeee..837213fd3 100644 --- a/app/org/maproulette/jobs/SchedulerActor.scala +++ b/app/org/maproulette/jobs/SchedulerActor.scala @@ -92,6 +92,8 @@ class SchedulerActor @Inject() ( this.handleArchiveChallenges(action) case RunJob("updateChallengeCompletionMetrics", action) => this.handleUpdateChallengeCompletionMetrics(action) + case RunJob("refreshTileAggregates", action) => + this.refreshTileAggregates(action) } /** @@ -168,11 +170,11 @@ class SchedulerActor @Inject() ( db.withTransaction { implicit c => val query = - s"""UPDATE challenges + s"""UPDATE challenges SET location = (SELECT ST_Centroid(ST_Collect(ST_Makevalid(location))) FROM tasks WHERE parent_id = ${id}), - bounding = (SELECT ST_Envelope(ST_Buffer((ST_SetSRID(ST_Extent(location), 4326))::geography,2)::geometry) + bounding = (SELECT ST_Envelope(ST_Expand(ST_SetSRID(ST_Extent(location), 4326), 0.0001)) FROM tasks WHERE parent_id = ${id}), last_updated = NOW(), @@ -876,6 +878,25 @@ class SchedulerActor @Inject() ( logger.warn(s"The KeepRight challenge creation failed. ${f.getMessage}") } } + + /** + * Rebuilds all pre-computed tile aggregates. + * Tiles are used for efficient map display of tasks at scale. + * + * @param action - action string + */ + def refreshTileAggregates(action: String): Unit = { + val start = System.currentTimeMillis + logger.info(s"Scheduled Task '$action': Starting full tile rebuild") + + val totalTiles = serviceManager.tileAggregate.rebuildAllTiles() + + val totalTime = System.currentTimeMillis - start + logger.info( + s"Scheduled Task '$action': Finished run. Time spent: ${totalTime}ms. " + + s"Total tiles: $totalTiles" + ) + } } object SchedulerActor { diff --git a/conf/application.conf b/conf/application.conf index 34a3f34bd..710b1148b 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -229,7 +229,7 @@ maproulette { cleanLocks.interval = "1 hour" cleanClaimLocks.interval = "1 hour" runChallengeSchedules.interval = "24 hours" - updateLocations.interval = "2 hours" + # updateLocations.interval = "2 hours" # Disabled for testing cleanOldTasks { interval = "24 hours" olderThan = "31 days" @@ -292,6 +292,9 @@ maproulette { interval = "7 days" startTime = "1:00:00" # Snapshot every week at 1am local time } + tileRefresh { + interval = "5 minutes" + } } mapillary { host = "a.mapillary.com" diff --git a/conf/evolutions/default/107.sql b/conf/evolutions/default/107.sql new file mode 100644 index 000000000..8e694115d --- /dev/null +++ b/conf/evolutions/default/107.sql @@ -0,0 +1,231 @@ +# --- !Ups + +-- Table for storing pre-computed task groups at all zoom levels (0-14) +-- Zoom 0-13: One entry per tile - can be single, overlapping, or cluster +-- - Single (1 task at 1 location): group_type=0, has task_ids +-- - Overlapping (N tasks at 1 location): group_type=1, has task_ids +-- - Cluster (N tasks at multiple locations): group_type=2, no task_ids +-- Zoom 14: One entry per overlap group (for frontend clustering at 14-22) +-- - Single: group_type=0, has task_ids +-- - Overlapping: group_type=1, has task_ids +CREATE TABLE tile_task_groups ( + id SERIAL PRIMARY KEY, + z SMALLINT NOT NULL, + x INTEGER NOT NULL, + y INTEGER NOT NULL, + group_type SMALLINT NOT NULL, -- 0=single task, 1=overlapping tasks, 2=cluster + centroid_lat DOUBLE PRECISION NOT NULL, + centroid_lng DOUBLE PRECISION NOT NULL, + task_ids BIGINT[] NOT NULL, + task_count INTEGER NOT NULL, + counts_by_filter JSONB DEFAULT '{}'::jsonb, + last_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW() +);; + +CREATE INDEX idx_tile_task_groups_coords ON tile_task_groups (z, x, y);; +CREATE INDEX idx_tile_task_groups_zoom ON tile_task_groups (z) WHERE task_count > 0;; + +-- Coordinate conversion functions +CREATE FUNCTION lng_to_tile_x(lng DOUBLE PRECISION, zoom INTEGER) RETURNS INTEGER AS $$ + SELECT FLOOR((lng + 180.0) / 360.0 * (1 << zoom))::INTEGER +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;; + +CREATE FUNCTION lat_to_tile_y(lat DOUBLE PRECISION, zoom INTEGER) RETURNS INTEGER AS $$ + SELECT FLOOR((1.0 - LN(TAN(RADIANS(GREATEST(-85.0511, LEAST(85.0511, lat)))) + + 1.0 / COS(RADIANS(GREATEST(-85.0511, LEAST(85.0511, lat))))) / PI()) / 2.0 * (1 << zoom))::INTEGER +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;; + +-- Function to rebuild a specific zoom level +CREATE FUNCTION rebuild_zoom_level(p_zoom INTEGER) RETURNS INTEGER AS $$ +DECLARE + groups_created INTEGER;; + start_time TIMESTAMP;; + deleted_count INTEGER;; + effective_zoom INTEGER;; +BEGIN + -- Offset zoom by +2 for tile calculations (zoom 0 uses zoom 2 grid, etc.) + effective_zoom := p_zoom + 2;; + start_time := clock_timestamp();; + RAISE NOTICE 'Zoom %: Starting rebuild...', p_zoom;; + + DELETE FROM tile_task_groups WHERE z = p_zoom;; + GET DIAGNOSTICS deleted_count = ROW_COUNT;; + RAISE NOTICE 'Zoom %: Deleted % existing groups', p_zoom, deleted_count;; + + IF p_zoom = 14 THEN + -- Zoom 14: Full detail - one entry per overlap group with task_ids + -- Frontend will handle clustering for zoom levels 14-22 + INSERT INTO tile_task_groups (z, x, y, group_type, centroid_lat, centroid_lng, task_ids, task_count, counts_by_filter) + WITH task_clusters AS ( + SELECT + t.id as task_id, + ST_Y(t.location) as lat, + ST_X(t.location) as lng, + lng_to_tile_x(ST_X(t.location), p_zoom) as tile_x, + lat_to_tile_y(ST_Y(t.location), p_zoom) as tile_y, + COALESCE(c.difficulty, 0) as difficulty, + COALESCE(c.is_global, false) as is_global, + ST_ClusterDBSCAN(t.location, eps := 0.000001, minpoints := 1) OVER ( + PARTITION BY lng_to_tile_x(ST_X(t.location), p_zoom), lat_to_tile_y(ST_Y(t.location), p_zoom) + ) as cluster_id + FROM tasks t + INNER JOIN challenges c ON c.id = t.parent_id + INNER JOIN projects p ON p.id = c.parent_id + WHERE t.location IS NOT NULL + AND NOT ST_IsEmpty(t.location) + AND ST_X(t.location) IS NOT NULL + AND ST_Y(t.location) IS NOT NULL + AND t.status IN (0, 3, 6) + AND c.deleted = FALSE AND c.enabled = TRUE AND c.is_archived = FALSE + AND p.deleted = FALSE AND p.enabled = TRUE + ), + grouped_tasks AS ( + SELECT + tile_x, + tile_y, + cluster_id, + AVG(lat) as centroid_lat, + AVG(lng) as centroid_lng, + ARRAY_AGG(task_id ORDER BY task_id) as task_ids, + COUNT(*)::INTEGER as task_count, + jsonb_build_object( + 'd1_gf', COUNT(*) FILTER (WHERE difficulty = 1 AND NOT is_global), + 'd1_gt', COUNT(*) FILTER (WHERE difficulty = 1 AND is_global), + 'd2_gf', COUNT(*) FILTER (WHERE difficulty = 2 AND NOT is_global), + 'd2_gt', COUNT(*) FILTER (WHERE difficulty = 2 AND is_global), + 'd3_gf', COUNT(*) FILTER (WHERE difficulty = 3 AND NOT is_global), + 'd3_gt', COUNT(*) FILTER (WHERE difficulty = 3 AND is_global), + 'd0_gf', COUNT(*) FILTER (WHERE difficulty NOT IN (1,2,3) AND NOT is_global), + 'd0_gt', COUNT(*) FILTER (WHERE difficulty NOT IN (1,2,3) AND is_global) + ) as counts_by_filter + FROM task_clusters + GROUP BY tile_x, tile_y, cluster_id + ) + SELECT + p_zoom as z, + tile_x as x, + tile_y as y, + CASE WHEN task_count = 1 THEN 0 ELSE 1 END as group_type, -- 0=single, 1=overlapping + centroid_lat, + centroid_lng, + task_ids, + task_count, + counts_by_filter + FROM grouped_tasks;; + + GET DIAGNOSTICS groups_created = ROW_COUNT;; + RAISE NOTICE 'Zoom %: Created % task/overlap groups (full detail for frontend clustering)', p_zoom, groups_created;; + ELSE + -- Zoom 0-13: One entry per tile + -- Detect if tile has one location (single/overlap) or multiple locations (cluster) + INSERT INTO tile_task_groups (z, x, y, group_type, centroid_lat, centroid_lng, task_ids, task_count, counts_by_filter) + WITH tile_tasks AS ( + SELECT + t.id as task_id, + ST_Y(t.location) as lat, + ST_X(t.location) as lng, + lng_to_tile_x(ST_X(t.location), effective_zoom) as tile_x, + lat_to_tile_y(ST_Y(t.location), effective_zoom) as tile_y, + COALESCE(c.difficulty, 0) as difficulty, + COALESCE(c.is_global, false) as is_global, + -- Detect overlaps within the tile (~0.1 meter precision) + ST_ClusterDBSCAN(t.location, eps := 0.000001, minpoints := 1) OVER ( + PARTITION BY lng_to_tile_x(ST_X(t.location), effective_zoom), lat_to_tile_y(ST_Y(t.location), effective_zoom) + ) as overlap_cluster_id + FROM tasks t + INNER JOIN challenges c ON c.id = t.parent_id + INNER JOIN projects p ON p.id = c.parent_id + WHERE t.location IS NOT NULL + AND NOT ST_IsEmpty(t.location) + AND ST_X(t.location) IS NOT NULL + AND ST_Y(t.location) IS NOT NULL + AND t.status IN (0, 3, 6) + AND c.deleted = FALSE AND c.enabled = TRUE AND c.is_archived = FALSE + AND p.deleted = FALSE AND p.enabled = TRUE + ), + tile_summary AS ( + SELECT + tile_x, + tile_y, + COUNT(DISTINCT overlap_cluster_id)::INTEGER as num_locations, + AVG(lat) as centroid_lat, + AVG(lng) as centroid_lng, + ARRAY_AGG(task_id ORDER BY task_id) as task_ids, + COUNT(*)::INTEGER as task_count, + jsonb_build_object( + 'd1_gf', COUNT(*) FILTER (WHERE difficulty = 1 AND NOT is_global), + 'd1_gt', COUNT(*) FILTER (WHERE difficulty = 1 AND is_global), + 'd2_gf', COUNT(*) FILTER (WHERE difficulty = 2 AND NOT is_global), + 'd2_gt', COUNT(*) FILTER (WHERE difficulty = 2 AND is_global), + 'd3_gf', COUNT(*) FILTER (WHERE difficulty = 3 AND NOT is_global), + 'd3_gt', COUNT(*) FILTER (WHERE difficulty = 3 AND is_global), + 'd0_gf', COUNT(*) FILTER (WHERE difficulty NOT IN (1,2,3) AND NOT is_global), + 'd0_gt', COUNT(*) FILTER (WHERE difficulty NOT IN (1,2,3) AND is_global) + ) as counts_by_filter + FROM tile_tasks + GROUP BY tile_x, tile_y + ) + SELECT + p_zoom as z, + tile_x as x, + tile_y as y, + CASE + WHEN num_locations = 1 AND task_count = 1 THEN 0 -- single task (isolated) + WHEN num_locations = 1 THEN 1 -- overlapping tasks (same location) + ELSE 2 -- cluster (multiple locations) + END as group_type, + centroid_lat, + centroid_lng, + CASE + WHEN num_locations = 1 THEN task_ids -- single/overlap: store task_ids + ELSE ARRAY[]::BIGINT[] -- cluster: no task_ids + END as task_ids, + task_count, + counts_by_filter + FROM tile_summary;; + + GET DIAGNOSTICS groups_created = ROW_COUNT;; + RAISE NOTICE 'Zoom %: Created % tile entries (singles/overlaps/clusters)', p_zoom, groups_created;; + END IF;; + + RAISE NOTICE 'Zoom %: Completed in % ms', p_zoom, EXTRACT(MILLISECONDS FROM (clock_timestamp() - start_time))::INTEGER;; + RETURN groups_created;; +END +$$ LANGUAGE plpgsql;; + +-- Function to rebuild all zoom levels +CREATE FUNCTION rebuild_all_tile_aggregates() RETURNS TABLE(zoom_level INTEGER, tiles_created INTEGER) AS $$ +DECLARE + total_start TIMESTAMP;; + total_groups INTEGER := 0;; +BEGIN + total_start := clock_timestamp();; + RAISE NOTICE '=== Starting full tile aggregate rebuild ===';; + RAISE NOTICE 'Zoom 0-13: Singles, overlaps, or clusters per tile';; + RAISE NOTICE 'Zoom 14: Individual tasks + overlaps (for frontend clustering at 14-22)';; + RAISE NOTICE '';; + + FOR zoom_level IN 0..14 LOOP + tiles_created := rebuild_zoom_level(zoom_level);; + total_groups := total_groups + tiles_created;; + RETURN NEXT;; + END LOOP;; + + RAISE NOTICE '';; + RAISE NOTICE '=== Rebuild complete ===';; + RAISE NOTICE 'Total groups created: %', total_groups;; + RAISE NOTICE 'Total time: % ms', EXTRACT(MILLISECONDS FROM (clock_timestamp() - total_start))::INTEGER;; +END +$$ LANGUAGE plpgsql;; + +-- Run after setup: SELECT * FROM rebuild_all_tile_aggregates(); + +# --- !Downs + +DROP FUNCTION IF EXISTS rebuild_all_tile_aggregates();; +DROP FUNCTION IF EXISTS rebuild_zoom_level(INTEGER);; +DROP FUNCTION IF EXISTS lat_to_tile_y(DOUBLE PRECISION, INTEGER);; +DROP FUNCTION IF EXISTS lng_to_tile_x(DOUBLE PRECISION, INTEGER);; +DROP INDEX IF EXISTS idx_tile_task_groups_zoom;; +DROP INDEX IF EXISTS idx_tile_task_groups_coords;; +DROP TABLE IF EXISTS tile_task_groups;; diff --git a/conf/v2_route/task.api b/conf/v2_route/task.api index 9d587ff53..b871fb510 100644 --- a/conf/v2_route/task.api +++ b/conf/v2_route/task.api @@ -678,6 +678,79 @@ PUT /markers/box/:left/:bottom/:right/:top @org.maproulette.framework GET /taskMarkers @org.maproulette.framework.controller.TaskController.getTaskMarkers(statuses: String, global:Boolean ?= false, cluster:Boolean ?= false, bounds: Option[String], location_id: Option[Long] ?= None, keywords:Option[String] ?= None, difficulty:Option[Int] ?= None) ### # tags: [ Task ] +# operationId: task_get_task_tiles +# summary: Get Task Tiles +# description: | +# Returns pre-computed tile aggregates for efficient map display at scale. +# Only includes tasks with status 0 (Created), 3 (Skipped), or 6 (Too Hard). +# For >= 2000 tasks, returns clusters. For < 2000 tasks, returns individual markers. +# +# Filter behavior: +# - difficulty & global: Filtered from pre-computed tile data (fast) +# - location_id: Queries tasks within polygon, clusters if > 2000 +# - keywords: Falls back to dynamic query (not pre-computed) +# responses: +# '200': +# description: Task markers with clusters and/or individual tasks (including overlaps) +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/org.maproulette.framework.model.TaskMarkerResponse' +# parameters: +# - name: z +# in: path +# description: Zoom level (0-14 for pre-computed tiles) +# required: true +# schema: +# type: integer +# minimum: 0 +# maximum: 20 +# - name: bounds +# in: query +# description: Comma-separated bounding box coordinates (left,bottom,right,top) +# required: true +# schema: +# type: string +# example: "-122.5,37.5,-122.0,38.0" +# - name: global +# in: query +# description: Include global challenges (filtered from pre-computed data) +# required: false +# schema: +# type: boolean +# default: false +# - name: location_id +# in: query +# description: Nominatim place_id for polygon filtering +# required: false +# schema: +# type: integer +# - name: keywords +# in: query +# description: Comma-separated keywords to filter by (triggers dynamic query) +# required: false +# schema: +# type: string +# - name: difficulty +# in: query +# description: Filter by difficulty (filtered from pre-computed data) +# required: false +# schema: +# type: integer +# enum: [1, 2, 3] +### +GET /taskTiles/:z @org.maproulette.framework.controller.TaskController.getTaskTiles(z: Int, bounds: String, global: Boolean ?= false, location_id: Option[Long] ?= None, keywords: Option[String] ?= None, difficulty: Option[Int] ?= None) +### +# tags: [ Task ] +# summary: Get Task Tile Data by Coordinates +# description: Returns task data for a specific tile (z/x/y). Used for zoom 14+ where individual tiles can be cached. +# responses: +# '200': +# description: Task marker response for the tile +### +GET /taskTile/:z/:x/:y @org.maproulette.framework.controller.TaskController.getTaskTile(z: Int, x: Int, y: Int, global: Boolean ?= false, difficulty: Option[Int] ?= None) +### +# tags: [ Task ] # operationId: task_update_task_changeset # summary: Update Task Changeset # description: Will update the changeset of the task. It will do this by attempting to match the OSM changeset to the Task based on the geometry and the time that the changeset was executed. diff --git a/test/org/maproulette/utils/TestSpec.scala b/test/org/maproulette/utils/TestSpec.scala index 77dab0cc3..d0f568f34 100644 --- a/test/org/maproulette/utils/TestSpec.scala +++ b/test/org/maproulette/utils/TestSpec.scala @@ -147,6 +147,7 @@ trait TestSpec extends PlaySpec with MockitoSugar { val leaderboardService = mock[LeaderboardService] val taskHistoryService = mock[TaskHistoryService] val nominatimService = mock[NominatimService] + val tileAggregateService = mock[TileAggregateService] val serviceManager = new ServiceManager( Providers.of[ProjectService](projectService), Providers.of[GrantService](grantService), @@ -171,7 +172,8 @@ trait TestSpec extends PlaySpec with MockitoSugar { Providers.of[NotificationService](notificationService), Providers.of[LeaderboardService](leaderboardService), Providers.of[TaskHistoryService](taskHistoryService), - Providers.of[NominatimService](nominatimService) + Providers.of[NominatimService](nominatimService), + Providers.of[TileAggregateService](tileAggregateService) ) val permission = new Permission(Providers.of[DALManager](dalManager), serviceManager, new Config())