Implement tile aggregation system for efficient map display

app/org/maproulette/Config.scala

@@ -348,6 +348,8 @@ object Config {
   val KEY_SCHEDULER_SNAPSHOT_CHALLENGES_INTERVAL =
     s"$SUB_GROUP_SCHEDULER.challengesSnapshot.interval"
   val KEY_SCHEDULER_SNAPSHOT_CHALLENGES_START = s"$SUB_GROUP_SCHEDULER.challengesSnapshot.startTime"
+  val KEY_SCHEDULER_TILE_REFRESH_INTERVAL     = s"$SUB_GROUP_SCHEDULER.tileRefresh.interval"
+  val KEY_SCHEDULER_TILE_REFRESH_BATCH_SIZE   = s"$SUB_GROUP_SCHEDULER.tileRefresh.batchSize"
 
   val KEY_MAPROULETTE_FRONTEND = s"$GROUP_MAPROULETTE.frontend"
   val SUB_GROUP_MAPILLARY      = s"$GROUP_MAPROULETTE.mapillary"
@@ -392,6 +394,7 @@ object Config {
   val DEFAULT_MR3_HOST                                = "/external"
   val DEFAULT_VIRTUAL_CHALLENGE_LIMIT                 = 100
   val DEFAULT_VIRTUAL_CHALLENGE_BATCH_SIZE            = 500
+  val DEFAULT_TILE_REFRESH_BATCH_SIZE                 = 1000
   val DEFAULT_VIRTUAL_CHALLENGE_EXPIRY                = "6 hours"
   val DEFAULT_CHANGESET_HOUR_LIMIT                    = "1 hour"
   val DEFAULT_CHANGESET_ENABLED                       = false

app/org/maproulette/framework/controller/TaskController.scala

@@ -327,6 +327,60 @@ class TaskController @Inject() (
     }
   }
 
+  /**
+    * Gets task data using pre-computed tile aggregates for efficient map display at scale.
+    * Uses a tile pyramid system with pre-computed counts broken down by difficulty × global.
+    *
+    * Behavior by filter:
+    * - difficulty & global: Filtered from pre-computed tile data (fast)
+    * - location_id: Recursive tile drilling until within polygon or < 2000 tasks
+    * - keywords: Falls back to dynamic query (challenge-level filter, not pre-computed)
+    *
+    * All fetched data is re-clustered into ~80 clusters for display.
+    * When total tasks < 2000, returns individual task markers instead of clusters.
+    *
+    * @param z           Zoom level (0-14 for pre-computed tiles)
+    * @param bounds      Comma-separated bounding box: left,bottom,right,top
+    * @param global      Whether to include global challenges
+    * @param location_id Optional Nominatim place_id for polygon filtering
+    * @param keywords    Optional keywords filter (triggers fallback to dynamic query)
+    * @param difficulty  Optional difficulty filter (1=Easy, 2=Normal, 3=Expert)
+    * @return TaskMarkerResponse with totalCount and either clusters or tasks (with overlaps)
+    */
+  def getTaskTiles(
+      z: Int,
+      bounds: String,
+      global: Boolean,
+      location_id: Option[Long],
+      keywords: Option[String],
+      difficulty: Option[Int]
+  ): Action[AnyContent] = Action.async { implicit request =>
+    this.sessionManager.userAwareRequest { implicit user =>
+      val boundingBox = bounds.split(",").map(_.trim.toDouble).toList match {
+        case List(left, bottom, right, top) =>
+          SearchLocation(left, bottom, right, top)
+        case _ =>
+          SearchLocation(-180.0, -90.0, 180.0, 90.0)
+      }
+
+      // Delegate all logic to the TileAggregateService
+      // The service handles:
+      // - difficulty & global filtering from pre-computed tile breakdowns
+      // - location_id filtering with recursive tile drilling
+      // - keywords filtering via fallback to dynamic query
+      // - re-clustering into ~80 clusters for display
+      val response = this.serviceManager.tileAggregate.getTileData(
+        z,
+        boundingBox,
+        difficulty,
+        global,
+        location_id,
+        keywords
+      )
+      Ok(Json.toJson(response))
+    }
+  }
+
 // for getting more detailed task marker data on individul makrers
   // def getTaskMarkerData(id: Long): Action[AnyContent] = Action.async { implicit request =>
   //   this.sessionManager.userAwareRequest { implicit user =>

app/org/maproulette/framework/model/TileAggregate.scala

@@ -0,0 +1,150 @@
+/*
+ * Copyright (C) 2020 MapRoulette contributors (see CONTRIBUTORS.md).
+ * Licensed under the Apache License, Version 2.0 (see LICENSE).
+ */
+package org.maproulette.framework.model
+
+import play.api.libs.json._
+
+/**
+  * Represents a cluster point (centroid with count) for map display
+  *
+  * @param lat   Cluster centroid latitude
+  * @param lng   Cluster centroid longitude
+  * @param count Number of tasks in this cluster
+  */
+case class ClusterPoint(
+    lat: Double,
+    lng: Double,
+    count: Int
+)
+
+object ClusterPoint {
+  implicit val clusterPointFormat: Format[ClusterPoint] = Json.format[ClusterPoint]
+}
+
+/**
+  * Counts broken down by difficulty × global filter combinations.
+  * Keys are like "d1_gf" = difficulty 1, global false
+  *
+  * @param d1_gf Difficulty 1 (Easy), global false
+  * @param d1_gt Difficulty 1 (Easy), global true
+  * @param d2_gf Difficulty 2 (Normal), global false
+  * @param d2_gt Difficulty 2 (Normal), global true
+  * @param d3_gf Difficulty 3 (Expert), global false
+  * @param d3_gt Difficulty 3 (Expert), global true
+  * @param d0_gf Difficulty not set, global false
+  * @param d0_gt Difficulty not set, global true
+  */
+case class FilterCounts(
+    d1_gf: Int = 0,
+    d1_gt: Int = 0,
+    d2_gf: Int = 0,
+    d2_gt: Int = 0,
+    d3_gf: Int = 0,
+    d3_gt: Int = 0,
+    d0_gf: Int = 0,
+    d0_gt: Int = 0
+) {
+
+  /**
+    * Get count for specific difficulty and global filters
+    *
+    * @param difficulty Optional difficulty filter (1, 2, 3)
+    * @param global     Whether to include global challenges (true = all, false = non-global only)
+    * @return Filtered count
+    */
+  def getFilteredCount(difficulty: Option[Int], global: Boolean): Int = {
+    // global=true means "include global challenges" → return ALL tasks
+    // global=false means "exclude global challenges" → return only non-global (*_gf)
+    difficulty match {
+      case Some(1) => if (global) d1_gf + d1_gt else d1_gf
+      case Some(2) => if (global) d2_gf + d2_gt else d2_gf
+      case Some(3) => if (global) d3_gf + d3_gt else d3_gf
+      case None    =>
+        // No difficulty filter - sum all difficulties
+        if (global) d1_gf + d1_gt + d2_gf + d2_gt + d3_gf + d3_gt + d0_gf + d0_gt
+        else d1_gf + d2_gf + d3_gf + d0_gf
+      case _ =>
+        // Unknown difficulty - treat as "other"
+        if (global) d0_gf + d0_gt else d0_gf
+    }
+  }
+
+  /**
+    * Get total count (all combinations)
+    */
+  def total: Int = d1_gf + d1_gt + d2_gf + d2_gt + d3_gf + d3_gt + d0_gf + d0_gt
+}
+
+object FilterCounts {
+  implicit val filterCountsFormat: Format[FilterCounts] = Json.format[FilterCounts]
+
+  def fromJson(json: JsValue): FilterCounts = {
+    FilterCounts(
+      d1_gf = (json \ "d1_gf").asOpt[Int].getOrElse(0),
+      d1_gt = (json \ "d1_gt").asOpt[Int].getOrElse(0),
+      d2_gf = (json \ "d2_gf").asOpt[Int].getOrElse(0),
+      d2_gt = (json \ "d2_gt").asOpt[Int].getOrElse(0),
+      d3_gf = (json \ "d3_gf").asOpt[Int].getOrElse(0),
+      d3_gt = (json \ "d3_gt").asOpt[Int].getOrElse(0),
+      d0_gf = (json \ "d0_gf").asOpt[Int].getOrElse(0),
+      d0_gt = (json \ "d0_gt").asOpt[Int].getOrElse(0)
+    )
+  }
+}
+
+/**
+  * Represents a pre-computed tile aggregate for efficient map display at scale.
+  * Uses Web Mercator tile coordinates (z/x/y).
+  * Only tracks tasks with status 0, 3, or 6.
+  *
+  * @param z              Zoom level (0-14 for pre-computed)
+  * @param x              Tile X coordinate
+  * @param y              Tile Y coordinate
+  * @param taskCount      Total tasks in this tile (all filters)
+  * @param countsByFilter Counts broken down by difficulty × global
+  * @param centroidLat    Centroid latitude of all tasks in tile
+  * @param centroidLng    Centroid longitude of all tasks in tile
+  */
+case class TileAggregate(
+    z: Int,
+    x: Int,
+    y: Int,
+    taskCount: Int,
+    countsByFilter: FilterCounts,
+    centroidLat: Double,
+    centroidLng: Double
+) {
+
+  /**
+    * Get the filtered count for this tile based on difficulty and global filters
+    */
+  def getFilteredCount(difficulty: Option[Int], global: Boolean): Int = {
+    countsByFilter.getFilteredCount(difficulty, global)
+  }
+}
+
+object TileAggregate {
+  implicit val tileAggregateWrites: Writes[TileAggregate] = Json.writes[TileAggregate]
+  implicit val tileAggregateReads: Reads[TileAggregate]   = Json.reads[TileAggregate]
+}
+
+/**
+  * Response for tile-based queries with combined clusters.
+  * All data from tiles is combined and re-clustered into ~80 clusters.
+  *
+  * @param totalCount Total tasks matching the filter
+  * @param clusters   Combined clusters (target ~80)
+  * @param tasks      Individual task markers (when total < threshold)
+  */
+case class TileDataResponse(
+    totalCount: Int,
+    clusters: Option[List[ClusterPoint]] = None,
+    tasks: Option[List[TaskMarker]] = None
+)
+
+object TileDataResponse {
+  implicit val tileDataResponseWrites: Writes[TileDataResponse] = Json.writes[TileDataResponse]
+  implicit val tileDataResponseReads: Reads[TileDataResponse]   = Json.reads[TileDataResponse]
+}