From 19ebc53c80335c10e9cd6128d7e73927b7816d25 Mon Sep 17 00:00:00 2001 From: Serein Pfeiffer Date: Fri, 23 Jan 2026 13:03:51 +0100 Subject: [PATCH 01/38] Default withAttrLayers to true for GeoJsonFolder --- docs/mapget-config.md | 2 +- libs/http-service/src/cli.cpp | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/mapget-config.md b/docs/mapget-config.md index d44fddb2..ee94de5a 100644 --- a/docs/mapget-config.md +++ b/docs/mapget-config.md @@ -203,7 +203,7 @@ Required fields: Optional fields: -- `withAttrLayers`: boolean flag. If `true`, nested objects in the GeoJSON `properties` are interpreted as attribute layers; if `false`, only scalar top‑level properties are emitted. +- `withAttrLayers` (default: `true`): boolean flag. If `true`, nested objects in the GeoJSON `properties` are converted to mapget attribute layers; if `false`, only scalar top‑level properties are emitted and nested objects are silently dropped. Example: diff --git a/libs/http-service/src/cli.cpp b/libs/http-service/src/cli.cpp index 7e51c540..58464881 100644 --- a/libs/http-service/src/cli.cpp +++ b/libs/http-service/src/cli.cpp @@ -94,7 +94,9 @@ nlohmann::json geoJsonFolderSchema() }}, {"withAttrLayers", { {"type", "boolean"}, - {"title", "With Attribute Layers"} + {"title", "With Attribute Layers"}, + {"description", "Convert nested GeoJSON property objects to mapget attribute layers. Default: true."}, + {"default", true} }} }}, {"required", nlohmann::json::array({"folder"})}, @@ -233,7 +235,7 @@ void registerDefaultDatasourceTypes() { "GeoJsonFolder", [](YAML::Node const& config) -> DataSource::Ptr { if (auto folder = config["folder"]) { - bool withAttributeLayers = false; + bool withAttributeLayers = true; if (auto withAttributeLayersNode = config["withAttrLayers"]) withAttributeLayers = withAttributeLayersNode.as(); return std::make_shared(folder.as(), withAttributeLayers); From 9008ae00285b79ded5394cbc64a636641e3655e2 Mon Sep 17 00:00:00 2001 From: Serein Pfeiffer Date: Mon, 26 Jan 2026 12:24:43 +0100 Subject: [PATCH 02/38] Add optional mapId config for GeoJsonFolder --- libs/http-service/src/cli.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/libs/http-service/src/cli.cpp b/libs/http-service/src/cli.cpp index 58464881..e305e24d 100644 --- a/libs/http-service/src/cli.cpp +++ b/libs/http-service/src/cli.cpp @@ -92,6 +92,11 @@ nlohmann::json geoJsonFolderSchema() {"title", "Folder"}, {"description", "Path to a folder containing GeoJSON tiles."} }}, + {"mapId", { + {"type", "string"}, + {"title", "Map ID"}, + {"description", "Custom map identifier. If not provided, derived from folder path."} + }}, {"withAttrLayers", { {"type", "boolean"}, {"title", "With Attribute Layers"}, @@ -238,7 +243,10 @@ void registerDefaultDatasourceTypes() { bool withAttributeLayers = true; if (auto withAttributeLayersNode = config["withAttrLayers"]) withAttributeLayers = withAttributeLayersNode.as(); - return std::make_shared(folder.as(), withAttributeLayers); + std::string mapId; + if (auto mapIdNode = config["mapId"]) + mapId = mapIdNode.as(); + return std::make_shared(folder.as(), withAttributeLayers, mapId); } throw std::runtime_error("Missing `folder` field."); }, From b782cf3d48ef4bedaee03c758cab4cdfacc72775 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Wed, 14 Jan 2026 17:55:35 +0100 Subject: [PATCH 03/38] Add cached tile size breakdown stats --- .../model/include/mapget/model/featurelayer.h | 4 + libs/model/src/featurelayer.cpp | 92 +++++++++++++++++++ libs/service/include/mapget/service/cache.h | 5 + .../service/include/mapget/service/memcache.h | 7 +- .../include/mapget/service/nullcache.h | 5 +- .../include/mapget/service/sqlitecache.h | 3 +- libs/service/src/memcache.cpp | 10 +- libs/service/src/nullcache.cpp | 7 +- libs/service/src/service.cpp | 78 +++++++++++++++- libs/service/src/sqlitecache.cpp | 28 +++++- 10 files changed, 231 insertions(+), 8 deletions(-) diff --git a/libs/model/include/mapget/model/featurelayer.h b/libs/model/include/mapget/model/featurelayer.h index 97c4f569..07888c60 100644 --- a/libs/model/include/mapget/model/featurelayer.h +++ b/libs/model/include/mapget/model/featurelayer.h @@ -20,6 +20,7 @@ #include "geometry.h" #include "sourcedatareference.h" #include "pointnode.h" +#include "nlohmann/json.hpp" namespace mapget { @@ -209,6 +210,9 @@ class TileFeatureLayer : public TileLayer, public simfil::ModelPool /** Convert to (Geo-) JSON. */ nlohmann::json toJson() const override; + /** Report serialized size stats for feature-layer data and model-pool columns. */ + [[nodiscard]] nlohmann::json serializationSizeStats() const; + /** Access number of stored features */ size_t size() const; diff --git a/libs/model/src/featurelayer.cpp b/libs/model/src/featurelayer.cpp index 826883b4..cf891b66 100644 --- a/libs/model/src/featurelayer.cpp +++ b/libs/model/src/featurelayer.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -76,6 +77,39 @@ namespace throw std::out_of_range("Size out of range"); return (index << SourceAddressArenaSizeBits) | size; } + + class CountingStreambuf : public std::streambuf + { + public: + size_t size() const { return size_; } + + protected: + std::streamsize xsputn(const char* /*s*/, std::streamsize count) override + { + size_ += static_cast(count); + return count; + } + + int overflow(int ch) override + { + if (ch != EOF) + ++size_; + return ch; + } + + private: + size_t size_ = 0; + }; + + template + size_t measureBytes(Fn&& fn) + { + CountingStreambuf buf; + std::ostream os(&buf); + bitsery::Serializer s(os); + fn(s); + return buf.size(); + } } namespace mapget @@ -841,6 +875,64 @@ nlohmann::json TileFeatureLayer::toJson() const return result; } +nlohmann::json TileFeatureLayer::serializationSizeStats() const +{ + constexpr size_t maxColumnSize = std::numeric_limits::max(); + auto featureLayer = nlohmann::json::object(); + + featureLayer["features"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->features_, maxColumnSize); })); + featureLayer["attributes"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->attributes_, maxColumnSize); })); + featureLayer["validities"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->validities_, maxColumnSize); })); + featureLayer["feature-ids"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->featureIds_, maxColumnSize); })); + featureLayer["attribute-layers"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->attrLayers_, maxColumnSize); })); + featureLayer["attribute-layer-lists"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->attrLayerLists_, maxColumnSize); })); + featureLayer["feature-id-prefix"] = static_cast(measureBytes( + [&](auto& s) { s.object(impl_->featureIdPrefix_); })); + featureLayer["relations"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->relations_, maxColumnSize); })); + featureLayer["feature-hash-index"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->featureHashIndex_, maxColumnSize); })); + featureLayer["geometries"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->geom_, maxColumnSize); })); + featureLayer["point-buffers"] = static_cast(measureBytes( + [&](auto& s) { s.ext(impl_->pointBuffers_, bitsery::ext::ArrayArenaExt{}); })); + featureLayer["source-data-references"] = static_cast(measureBytes( + [&](auto& s) { s.container(impl_->sourceDataReferences_, maxColumnSize); })); + + int64_t featureLayerTotal = 0; + for (const auto& [_, value] : featureLayer.items()) { + if (value.is_number_integer()) + featureLayerTotal += value.get(); + } + + auto modelStats = ModelPool::serializationSizeStats(); + auto modelPool = nlohmann::json::object({ + {"roots", static_cast(modelStats.rootsBytes)}, + {"int64", static_cast(modelStats.int64Bytes)}, + {"double", static_cast(modelStats.doubleBytes)}, + {"string-data", static_cast(modelStats.stringDataBytes)}, + {"string-ranges", static_cast(modelStats.stringRangeBytes)}, + {"object-members", static_cast(modelStats.objectMemberBytes)}, + {"array-members", static_cast(modelStats.arrayMemberBytes)}, + }); + + int64_t modelPoolTotal = static_cast(modelStats.totalBytes()); + + return { + {"feature-layer", featureLayer}, + {"model-pool", modelPool}, + {"feature-layer-total-bytes", featureLayerTotal}, + {"model-pool-total-bytes", modelPoolTotal}, + {"total-bytes", featureLayerTotal + modelPoolTotal} + }; +} + size_t TileFeatureLayer::size() const { return numRoots(); diff --git a/libs/service/include/mapget/service/cache.h b/libs/service/include/mapget/service/cache.h index bee02b1d..50d92a28 100644 --- a/libs/service/include/mapget/service/cache.h +++ b/libs/service/include/mapget/service/cache.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "mapget/model/info.h" #include "mapget/model/featurelayer.h" @@ -29,6 +30,7 @@ class Cache : public TileLayerStream::StringPoolCache, public std::enable_shared }; using Ptr = std::shared_ptr; + using TileBlobVisitor = std::function; // The following methods are already implemented, // they forward to the virtual methods on-demand. @@ -52,6 +54,9 @@ class Cache : public TileLayerStream::StringPoolCache, public std::enable_shared /** Abstract: Upsert (update or insert) a TileLayer blob. */ virtual void putTileLayerBlob(MapTileKey const& k, std::string const& v) = 0; + /** Abstract: Iterate through all cached tile layer blobs. */ + virtual void forEachTileLayerBlob(const TileBlobVisitor& cb) const = 0; + /** Abstract: Retrieve a string-pool blob for a sourceNodeId. */ virtual std::optional getStringPoolBlob(std::string_view const& sourceNodeId) = 0; diff --git a/libs/service/include/mapget/service/memcache.h b/libs/service/include/mapget/service/memcache.h index 283c0e11..bdc2b452 100644 --- a/libs/service/include/mapget/service/memcache.h +++ b/libs/service/include/mapget/service/memcache.h @@ -29,6 +29,9 @@ class MemCache : public Cache /** Upsert a TileLayer blob. */ void putTileLayerBlob(MapTileKey const& k, std::string const& v) override; + /** Iterate over cached tile layer blobs. */ + void forEachTileLayerBlob(const TileBlobVisitor& cb) const override; + /** Retrieve a string-pool blob for a sourceNodeId -> No-Op */ std::optional getStringPoolBlob(std::string_view const& sourceNodeId) override {return {};} @@ -40,10 +43,10 @@ class MemCache : public Cache private: // Cached tile blobs. - std::shared_mutex cacheMutex_; + mutable std::shared_mutex cacheMutex_; std::unordered_map cachedTiles_; std::deque fifo_; uint32_t maxCachedTiles_ = 0; }; -} \ No newline at end of file +} diff --git a/libs/service/include/mapget/service/nullcache.h b/libs/service/include/mapget/service/nullcache.h index 00aa8e1a..66e29b71 100644 --- a/libs/service/include/mapget/service/nullcache.h +++ b/libs/service/include/mapget/service/nullcache.h @@ -20,6 +20,9 @@ class NullCache : public Cache /** Upsert a TileLayer blob - does nothing. */ void putTileLayerBlob(MapTileKey const& k, std::string const& v) override; + /** Iterate cached tile blobs - no-op. */ + void forEachTileLayerBlob(const TileBlobVisitor& cb) const override; + /** Retrieve a string-pool blob for a sourceNodeId - always returns empty. */ std::optional getStringPoolBlob(std::string_view const& sourceNodeId) override; @@ -27,4 +30,4 @@ class NullCache : public Cache void putStringPoolBlob(std::string_view const& sourceNodeId, std::string const& v) override; }; -} \ No newline at end of file +} diff --git a/libs/service/include/mapget/service/sqlitecache.h b/libs/service/include/mapget/service/sqlitecache.h index 55b42879..85387dcd 100644 --- a/libs/service/include/mapget/service/sqlitecache.h +++ b/libs/service/include/mapget/service/sqlitecache.h @@ -25,6 +25,7 @@ class SQLiteCache : public Cache std::optional getTileLayerBlob(MapTileKey const& k) override; void putTileLayerBlob(MapTileKey const& k, std::string const& v) override; + void forEachTileLayerBlob(const TileBlobVisitor& cb) const override; std::optional getStringPoolBlob(std::string_view const& sourceNodeId) override; void putStringPoolBlob(std::string_view const& sourceNodeId, std::string const& v) override; @@ -53,4 +54,4 @@ class SQLiteCache : public Cache } stmts_; }; -} // namespace mapget \ No newline at end of file +} // namespace mapget diff --git a/libs/service/src/memcache.cpp b/libs/service/src/memcache.cpp index d9a49dc8..f3d95a88 100644 --- a/libs/service/src/memcache.cpp +++ b/libs/service/src/memcache.cpp @@ -31,6 +31,14 @@ void MemCache::putTileLayerBlob(const MapTileKey& k, const std::string& v) } } +void MemCache::forEachTileLayerBlob(const TileBlobVisitor& cb) const +{ + std::shared_lock cacheLock(cacheMutex_); + for (const auto& [key, value] : cachedTiles_) { + cb(MapTileKey(key), value); + } +} + nlohmann::json MemCache::getStatistics() const { auto result = Cache::getStatistics(); result["memcache-map-size"] = (int64_t)cachedTiles_.size(); @@ -38,4 +46,4 @@ nlohmann::json MemCache::getStatistics() const { return result; } -} \ No newline at end of file +} diff --git a/libs/service/src/nullcache.cpp b/libs/service/src/nullcache.cpp index 0862a5ea..e1dc99e6 100644 --- a/libs/service/src/nullcache.cpp +++ b/libs/service/src/nullcache.cpp @@ -13,6 +13,11 @@ void NullCache::putTileLayerBlob(MapTileKey const& k, std::string const& v) // Do nothing - no caching } +void NullCache::forEachTileLayerBlob(const TileBlobVisitor& cb) const +{ + // No cached tiles. +} + std::optional NullCache::getStringPoolBlob(std::string_view const& sourceNodeId) { return std::nullopt; @@ -23,4 +28,4 @@ void NullCache::putStringPoolBlob(std::string_view const& sourceNodeId, std::str // Do nothing - no caching } -} \ No newline at end of file +} diff --git a/libs/service/src/service.cpp b/libs/service/src/service.cpp index c8449d8c..9f0b1cbf 100644 --- a/libs/service/src/service.cpp +++ b/libs/service/src/service.cpp @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include "simfil/types.h" @@ -673,10 +675,84 @@ nlohmann::json Service::getStatistics() const }); } - return { + auto result = nlohmann::json{ {"datasources", datasources}, {"active-requests", impl_->requests_.size()} }; + + auto layerInfoByMap = std::unordered_map>>{}; + for (auto const& [_, info] : impl_->dataSourceInfo_) { + auto& layers = layerInfoByMap[info.mapId_]; + for (auto const& [layerId, layerInfo] : info.layers_) { + layers[layerId] = layerInfo; + } + } + + auto resolveLayerInfo = [&](std::string_view mapId, std::string_view layerId) -> std::shared_ptr { + auto mapIt = layerInfoByMap.find(std::string(mapId)); + if (mapIt == layerInfoByMap.end()) + return std::make_shared(); + auto layerIt = mapIt->second.find(std::string(layerId)); + if (layerIt == mapIt->second.end()) { + auto fallback = std::make_shared(); + fallback->layerId_ = std::string(layerId); + return fallback; + } + return layerIt->second; + }; + + int64_t parsedTiles = 0; + int64_t totalTileBytes = 0; + int64_t parseErrors = 0; + auto featureLayerTotals = nlohmann::json::object(); + auto modelPoolTotals = nlohmann::json::object(); + + auto addTotals = [](nlohmann::json& totals, const nlohmann::json& stats) { + for (const auto& [key, value] : stats.items()) { + if (!value.is_number_integer()) + continue; + totals[key] = totals.value(key, 0) + value.get(); + } + }; + + impl_->cache_->forEachTileLayerBlob( + [&](const MapTileKey& key, const std::string& blob) + { + if (key.layer_ != LayerType::Features) + return; + ++parsedTiles; + totalTileBytes += static_cast(blob.size()); + + try { + std::istringstream inputStream(blob, std::ios::binary); + auto tile = std::make_shared( + inputStream, + [&](auto&& mapId, auto&& layerId) { + return resolveLayerInfo(mapId, layerId); + }, + [&](auto&& nodeId) { + return impl_->cache_->getStringPool(nodeId); + }); + auto sizeStats = tile->serializationSizeStats(); + addTotals(featureLayerTotals, sizeStats["feature-layer"]); + addTotals(modelPoolTotals, sizeStats["model-pool"]); + } + catch (const std::exception&) { + ++parseErrors; + } + }); + + if (parsedTiles > 0) { + result["cached-feature-tree-bytes"] = nlohmann::json{ + {"tile-count", parsedTiles}, + {"total-tile-bytes", totalTileBytes}, + {"parse-errors", parseErrors}, + {"feature-layer", featureLayerTotals}, + {"model-pool", modelPoolTotals} + }; + } + + return result; } } // namespace mapget diff --git a/libs/service/src/sqlitecache.cpp b/libs/service/src/sqlitecache.cpp index 74b2b308..c062e1ec 100644 --- a/libs/service/src/sqlitecache.cpp +++ b/libs/service/src/sqlitecache.cpp @@ -259,6 +259,32 @@ void SQLiteCache::putTileLayerBlob(MapTileKey const& k, std::string const& v) } } +void SQLiteCache::forEachTileLayerBlob(const TileBlobVisitor& cb) const +{ + std::lock_guard lock(dbMutex_); + + sqlite3_stmt* stmt = nullptr; + int rc = sqlite3_prepare_v2(db_, "SELECT key, data FROM tiles", -1, &stmt, nullptr); + if (rc != SQLITE_OK) { + raise(fmt::format("Failed to prepare tile iteration statement: {}", sqlite3_errmsg(db_))); + } + + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + const char* key = reinterpret_cast(sqlite3_column_text(stmt, 0)); + const void* data = sqlite3_column_blob(stmt, 1); + int size = sqlite3_column_bytes(stmt, 1); + if (key && data && size >= 0) { + cb(MapTileKey(key), std::string(static_cast(data), size)); + } + } + if (rc != SQLITE_DONE) { + sqlite3_finalize(stmt); + raise(fmt::format("Error iterating cached tiles: {}", sqlite3_errmsg(db_))); + } + + sqlite3_finalize(stmt); +} + void SQLiteCache::cleanupOldestTiles() { // Delete the oldest tile @@ -315,4 +341,4 @@ void SQLiteCache::putStringPoolBlob(std::string_view const& sourceNodeId, std::s } } -} // namespace mapget \ No newline at end of file +} // namespace mapget From 4aeec718bbdecc70503426493fa0384229794e5e Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 15 Jan 2026 11:04:21 +0100 Subject: [PATCH 04/38] Use simfil release branch. --- cmake/deps.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/deps.cmake b/cmake/deps.cmake index 7473f2b9..1ed0717c 100644 --- a/cmake/deps.cmake +++ b/cmake/deps.cmake @@ -15,7 +15,7 @@ CPMAddPackage( "EXPECTED_BUILD_TESTS OFF" "EXPECTED_BUILD_PACKAGE_DEB OFF") CPMAddPackage( - URI "gh:Klebert-Engineering/simfil@0.6.2" + URI "gh:Klebert-Engineering/simfil@0.6.3#v0.6.3" OPTIONS "SIMFIL_WITH_MODEL_JSON ON" "SIMFIL_SHARED OFF") From 0659ee8ca0137ba97fb2c695f2673cb613320371 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 15 Jan 2026 12:15:51 +0100 Subject: [PATCH 05/38] Use TileLayerStream in Cache statistics generation. --- libs/service/src/service.cpp | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/libs/service/src/service.cpp b/libs/service/src/service.cpp index 9f0b1cbf..4abbf9bd 100644 --- a/libs/service/src/service.cpp +++ b/libs/service/src/service.cpp @@ -709,12 +709,28 @@ nlohmann::json Service::getStatistics() const auto addTotals = [](nlohmann::json& totals, const nlohmann::json& stats) { for (const auto& [key, value] : stats.items()) { - if (!value.is_number_integer()) - continue; - totals[key] = totals.value(key, 0) + value.get(); + if (value.is_number_integer()) + { + totals[key] = totals.value(key, 0) + value.get(); + } + else if (value.is_number_float()) + { + totals[key] = totals.value(key, .0) + value.get(); + } } }; + TileLayerStream::Reader tileReader( + resolveLayerInfo, + [&](auto&& parsedLayer) + { + auto tile = std::static_pointer_cast(parsedLayer); + auto sizeStats = tile->serializationSizeStats(); + addTotals(featureLayerTotals, sizeStats["feature-layer"]); + addTotals(modelPoolTotals, sizeStats["model-pool"]); + }, + impl_->cache_); + impl_->cache_->forEachTileLayerBlob( [&](const MapTileKey& key, const std::string& blob) { @@ -722,20 +738,8 @@ nlohmann::json Service::getStatistics() const return; ++parsedTiles; totalTileBytes += static_cast(blob.size()); - try { - std::istringstream inputStream(blob, std::ios::binary); - auto tile = std::make_shared( - inputStream, - [&](auto&& mapId, auto&& layerId) { - return resolveLayerInfo(mapId, layerId); - }, - [&](auto&& nodeId) { - return impl_->cache_->getStringPool(nodeId); - }); - auto sizeStats = tile->serializationSizeStats(); - addTotals(featureLayerTotals, sizeStats["feature-layer"]); - addTotals(modelPoolTotals, sizeStats["model-pool"]); + tileReader.read(blob); } catch (const std::exception&) { ++parseErrors; From 516c2b6e3dcd62889b954976db4b26f7a2ce1dbd Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Wed, 21 Jan 2026 10:24:33 +0100 Subject: [PATCH 06/38] Introduce uWebSockets as dependency. --- cmake/deps.cmake | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/cmake/deps.cmake b/cmake/deps.cmake index 1ed0717c..9e22dc7a 100644 --- a/cmake/deps.cmake +++ b/cmake/deps.cmake @@ -64,6 +64,38 @@ if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) CPMAddPackage("gh:CLIUtils/CLI11@2.5.0") CPMAddPackage("gh:pboettch/json-schema-validator#2.3.0") CPMAddPackage("gh:okdshin/PicoSHA2@1.0.1") + + CPMAddPackage( + NAME uSockets + GIT_REPOSITORY https://github.com/uNetworking/uSockets + GIT_TAG v0.8.5 + GIT_SHALLOW ON + GIT_SUBMODULES "") + if (NOT TARGET uSockets) + file(GLOB_RECURSE U_SOCKETS_SOURCES "${uSockets_SOURCE_DIR}/src/*.c") + add_library(uSockets STATIC ${U_SOCKETS_SOURCES}) + target_include_directories(uSockets PUBLIC "${uSockets_SOURCE_DIR}/src") + target_compile_definitions(uSockets PRIVATE LIBUS_USE_OPENSSL) + target_link_libraries(uSockets PRIVATE OpenSSL::SSL OpenSSL::Crypto) + if (WIN32) + target_link_libraries(uSockets PRIVATE ws2_32) + endif() + endif() + + CPMAddPackage( + NAME uWebSockets + GIT_REPOSITORY https://github.com/uNetworking/uWebSockets + GIT_TAG v20.37.0 + GIT_SHALLOW ON + GIT_SUBMODULES "") + if (NOT TARGET uWebSockets) + add_library(uWebSockets INTERFACE) + target_include_directories(uWebSockets INTERFACE "${uWebSockets_SOURCE_DIR}/src") + target_link_libraries(uWebSockets INTERFACE uSockets ZLIB::ZLIB) + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") + target_compile_options(uWebSockets INTERFACE -Wno-deprecated-declarations) + endif() + endif() endif () if (MAPGET_WITH_WHEEL AND NOT TARGET pybind11) From fbc5d48060727451dc88a4150b94174d4727be03 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 22 Jan 2026 10:54:27 +0100 Subject: [PATCH 07/38] Backend transition: uWS for server, cpp-httplib for client. --- cmake/deps.cmake | 28 +- libs/http-datasource/CMakeLists.txt | 1 + .../include/mapget/detail/http-server.h | 9 +- .../http-datasource/datasource-server.h | 2 +- .../http-datasource/src/datasource-server.cpp | 175 ++-- libs/http-datasource/src/http-server.cpp | 335 +++++- libs/http-service/CMakeLists.txt | 1 + .../include/mapget/http-service/http-client.h | 2 +- .../mapget/http-service/http-service.h | 3 +- libs/http-service/src/http-client.cpp | 9 +- libs/http-service/src/http-service.cpp | 990 ++++++++++-------- libs/service/src/datasource.cpp | 7 +- 12 files changed, 1003 insertions(+), 559 deletions(-) diff --git a/cmake/deps.cmake b/cmake/deps.cmake index 9e22dc7a..a990b4a2 100644 --- a/cmake/deps.cmake +++ b/cmake/deps.cmake @@ -65,6 +65,19 @@ if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) CPMAddPackage("gh:pboettch/json-schema-validator#2.3.0") CPMAddPackage("gh:okdshin/PicoSHA2@1.0.1") + if (WIN32) + CPMAddPackage( + NAME libuv + GIT_REPOSITORY https://github.com/libuv/libuv + GIT_TAG v1.48.0 + GIT_SHALLOW ON + OPTIONS + "LIBUV_BUILD_TESTS OFF" + "LIBUV_BUILD_BENCH OFF" + "LIBUV_BUILD_SHARED OFF" + "LIBUV_BUILD_EXAMPLES OFF") + endif() + CPMAddPackage( NAME uSockets GIT_REPOSITORY https://github.com/uNetworking/uSockets @@ -72,13 +85,22 @@ if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) GIT_SHALLOW ON GIT_SUBMODULES "") if (NOT TARGET uSockets) - file(GLOB_RECURSE U_SOCKETS_SOURCES "${uSockets_SOURCE_DIR}/src/*.c") + file(GLOB_RECURSE U_SOCKETS_SOURCES CONFIGURE_DEPENDS + "${uSockets_SOURCE_DIR}/src/*.c" + "${uSockets_SOURCE_DIR}/src/*.cpp") add_library(uSockets STATIC ${U_SOCKETS_SOURCES}) target_include_directories(uSockets PUBLIC "${uSockets_SOURCE_DIR}/src") target_compile_definitions(uSockets PRIVATE LIBUS_USE_OPENSSL) - target_link_libraries(uSockets PRIVATE OpenSSL::SSL OpenSSL::Crypto) + target_link_libraries(uSockets PUBLIC OpenSSL::SSL OpenSSL::Crypto) if (WIN32) - target_link_libraries(uSockets PRIVATE ws2_32) + target_link_libraries(uSockets PUBLIC ws2_32) + if (TARGET uv_a) + target_link_libraries(uSockets PUBLIC uv_a) + elseif (TARGET uv) + target_link_libraries(uSockets PUBLIC uv) + else() + message(FATAL_ERROR "libuv was requested for uSockets on Windows, but no CMake target (uv_a/uv) was found.") + endif() endif() endif() diff --git a/libs/http-datasource/CMakeLists.txt b/libs/http-datasource/CMakeLists.txt index e46557b5..917eded2 100644 --- a/libs/http-datasource/CMakeLists.txt +++ b/libs/http-datasource/CMakeLists.txt @@ -18,6 +18,7 @@ target_include_directories(mapget-http-datasource target_link_libraries(mapget-http-datasource PUBLIC httplib::httplib + uWebSockets mapget-model mapget-service tiny-process-library) diff --git a/libs/http-datasource/include/mapget/detail/http-server.h b/libs/http-datasource/include/mapget/detail/http-server.h index c80b7b62..bcc9656f 100644 --- a/libs/http-datasource/include/mapget/detail/http-server.h +++ b/libs/http-datasource/include/mapget/detail/http-server.h @@ -3,9 +3,10 @@ #include #include -// Pre-declare httplib::Server to avoid including httplib.h in header -namespace httplib { -class Server; +// Forward declare uWebSockets app type to avoid including uWS headers in public headers. +namespace uWS { +template struct TemplatedApp; +using App = TemplatedApp; } namespace mapget { @@ -76,7 +77,7 @@ class HttpServer * This function is called upon the first call to go(), * and allows any derived server class to add endpoints. */ - virtual void setup(httplib::Server&) = 0; + virtual void setup(uWS::App&) = 0; /** * Derived servers can use this to control whether diff --git a/libs/http-datasource/include/mapget/http-datasource/datasource-server.h b/libs/http-datasource/include/mapget/http-datasource/datasource-server.h index ed11adde..501f01ac 100644 --- a/libs/http-datasource/include/mapget/http-datasource/datasource-server.h +++ b/libs/http-datasource/include/mapget/http-datasource/datasource-server.h @@ -50,7 +50,7 @@ class DataSourceServer : public HttpServer DataSourceInfo const& info(); private: - void setup(httplib::Server&) override; + void setup(uWS::App&) override; struct Impl; std::unique_ptr impl_; diff --git a/libs/http-datasource/src/datasource-server.cpp b/libs/http-datasource/src/datasource-server.cpp index f6ea10f0..a8435058 100644 --- a/libs/http-datasource/src/datasource-server.cpp +++ b/libs/http-datasource/src/datasource-server.cpp @@ -1,54 +1,51 @@ #include "datasource-server.h" -#include "mapget/detail/http-server.h" -#include "mapget/model/sourcedatalayer.h" -#include "mapget/model/featurelayer.h" + +#include "mapget/log.h" #include "mapget/model/info.h" -#include "mapget/model/layer.h" #include "mapget/model/stream.h" -#include "httplib.h" +#include + #include #include +#include + +#include "fmt/format.h" -namespace mapget { +namespace mapget +{ struct DataSourceServer::Impl { DataSourceInfo info_; - std::function tileFeatureCallback_ = [](auto&&) - { + std::function tileFeatureCallback_ = [](auto&&) { throw std::runtime_error("TileFeatureLayer callback is unset!"); }; - std::function tileSourceDataCallback_ = [](auto&&) - { + std::function tileSourceDataCallback_ = [](auto&&) { throw std::runtime_error("TileSourceDataLayer callback is unset!"); }; std::function(const LocateRequest&)> locateCallback_; std::shared_ptr strings_; - explicit Impl(DataSourceInfo info) - : info_(std::move(info)), strings_(std::make_shared(info_.nodeId_)) + explicit Impl(DataSourceInfo info) : info_(std::move(info)), strings_(std::make_shared(info_.nodeId_)) { } }; -DataSourceServer::DataSourceServer(DataSourceInfo const& info) - : HttpServer(), impl_(new Impl(info)) +DataSourceServer::DataSourceServer(DataSourceInfo const& info) : HttpServer(), impl_(new Impl(info)) { printPortToStdOut(true); } DataSourceServer::~DataSourceServer() = default; -DataSourceServer& -DataSourceServer::onTileFeatureRequest(std::function const& callback) +DataSourceServer& DataSourceServer::onTileFeatureRequest(std::function const& callback) { impl_->tileFeatureCallback_ = callback; return *this; } -DataSourceServer& -DataSourceServer::onTileSourceDataRequest(std::function const& callback) +DataSourceServer& DataSourceServer::onTileSourceDataRequest(std::function const& callback) { impl_->tileSourceDataCallback_ = callback; return *this; @@ -61,50 +58,49 @@ DataSourceServer& DataSourceServer::onLocateRequest( return *this; } -DataSourceInfo const& DataSourceServer::info() { - return impl_->info_; -} +DataSourceInfo const& DataSourceServer::info() { return impl_->info_; } -void DataSourceServer::setup(httplib::Server& server) +void DataSourceServer::setup(uWS::App& app) { - // Set up GET /tile endpoint - server.Get( - "/tile", - [this](const httplib::Request& req, httplib::Response& res) { - // Extract parameters from request. - auto layerIdParam = req.get_param_value("layer"); - auto layer = impl_->info_.getLayer(layerIdParam); - - auto tileIdParam = TileId{std::stoull(req.get_param_value("tileId"))}; + app.get("/tile", [this](auto* res, auto* req) { + try { + auto layerIdParam = req->getQuery("layer"); + auto tileIdParam = req->getQuery("tileId"); + + if (layerIdParam.empty() || tileIdParam.empty()) { + res->writeStatus("400 Bad Request"); + res->writeHeader("Content-Type", "text/plain"); + res->end("Missing query parameter: layer and/or tileId"); + return; + } + + auto layer = impl_->info_.getLayer(std::string(layerIdParam)); + + TileId tileId{std::stoull(std::string(tileIdParam))}; + auto stringPoolOffsetParam = (simfil::StringId)0; - if (req.has_param("stringPoolOffset")) - stringPoolOffsetParam = (simfil::StringId) - std::stoul(req.get_param_value("stringPoolOffset")); + auto stringPoolOffsetStr = req->getQuery("stringPoolOffset"); + if (!stringPoolOffsetStr.empty()) { + stringPoolOffsetParam = (simfil::StringId)std::stoul(std::string(stringPoolOffsetStr)); + } std::string responseType = "binary"; - if (req.has_param("responseType")) - responseType = req.get_param_value("responseType"); + auto responseTypeStr = req->getQuery("responseType"); + if (!responseTypeStr.empty()) { + responseType = std::string(responseTypeStr); + } - // Create response TileFeatureLayer. auto tileLayer = [&]() -> std::shared_ptr { switch (layer->type_) { case mapget::LayerType::Features: { auto tileFeatureLayer = std::make_shared( - tileIdParam, - impl_->info_.nodeId_, - impl_->info_.mapId_, - layer, - impl_->strings_); + tileId, impl_->info_.nodeId_, impl_->info_.mapId_, layer, impl_->strings_); impl_->tileFeatureCallback_(tileFeatureLayer); return tileFeatureLayer; } case mapget::LayerType::SourceData: { auto tileSourceLayer = std::make_shared( - tileIdParam, - impl_->info_.nodeId_, - impl_->info_.mapId_, - layer, - impl_->strings_); + tileId, impl_->info_.nodeId_, impl_->info_.mapId_, layer, impl_->strings_); impl_->tileSourceDataCallback_(tileSourceLayer); return tileSourceLayer; } @@ -113,45 +109,72 @@ void DataSourceServer::setup(httplib::Server& server) } }(); - // Serialize TileLayer using TileLayerStream. if (responseType == "binary") { - std::stringstream content; + std::string content; TileLayerStream::StringPoolOffsetMap stringPoolOffsets{ {impl_->info_.nodeId_, stringPoolOffsetParam}}; TileLayerStream::Writer layerWriter{ - [&](auto&& msg, auto&& msgType) { content << msg; }, + [&](std::string bytes, TileLayerStream::MessageType) { content.append(bytes); }, stringPoolOffsets}; layerWriter.write(tileLayer); - res.set_content(content.str(), "application/binary"); - } - else { - res.set_content(nlohmann::to_string(tileLayer->toJson()), "application/json"); - } - }); - - // Set up GET /info endpoint - server.Get( - "/info", - [this](const httplib::Request&, httplib::Response& res) { - nlohmann::json j = impl_->info_.toJson(); - res.set_content(j.dump(), "application/json"); - }); - // Set up POST /locate endpoint - server.Post( - "/locate", - [this](const httplib::Request& req, httplib::Response& res) { - LocateRequest parsedReq(nlohmann::json::parse(req.body)); - auto responseJson = nlohmann::json::array(); - - if (impl_->locateCallback_) { - for (auto const& response : impl_->locateCallback_(parsedReq)) { - responseJson.emplace_back(response.serialize()); - } + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "application/binary"); + res->end(content); + } else { + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "application/json"); + res->end(tileLayer->toJson().dump()); } + } + catch (std::exception const& e) { + res->writeStatus("500 Internal Server Error"); + res->writeHeader("Content-Type", "text/plain"); + res->end(std::string("Error: ") + e.what()); + } + }); + + app.get("/info", [this](auto* res, auto* /*req*/) { + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "application/json"); + res->end(impl_->info_.toJson().dump()); + }); + + app.post("/locate", [this](auto* res, auto* /*req*/) { + auto aborted = std::make_shared(false); + res->onAborted([aborted]() { *aborted = true; }); + + res->onData([this, res, aborted, body = std::string()](std::string_view chunk, bool last) mutable { + if (*aborted) + return; + body.append(chunk.data(), chunk.size()); + if (!last) + return; + try { + LocateRequest parsedReq(nlohmann::json::parse(body)); + auto responseJson = nlohmann::json::array(); + + if (impl_->locateCallback_) { + for (auto const& response : impl_->locateCallback_(parsedReq)) { + responseJson.emplace_back(response.serialize()); + } + } - res.set_content(responseJson.dump(), "application/json"); + if (*aborted) + return; + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "application/json"); + res->end(responseJson.dump()); + } + catch (std::exception const& e) { + if (*aborted) + return; + res->writeStatus("400 Bad Request"); + res->writeHeader("Content-Type", "text/plain"); + res->end(std::string("Invalid request: ") + e.what()); + } }); + }); } } // namespace mapget diff --git a/libs/http-datasource/src/http-server.cpp b/libs/http-datasource/src/http-server.cpp index 6b49eb66..cdceab2c 100644 --- a/libs/http-datasource/src/http-server.cpp +++ b/libs/http-datasource/src/http-server.cpp @@ -1,10 +1,23 @@ #include "mapget/detail/http-server.h" #include "mapget/log.h" -#include "httplib.h" -#include +#include +#include + #include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include #include "fmt/format.h" @@ -14,14 +27,120 @@ namespace mapget // initialize the atomic activeHttpServer with nullptr static std::atomic activeHttpServer = nullptr; +namespace +{ +struct MountPoint +{ + std::string urlPrefix; + std::filesystem::path fsRoot; +}; + +[[nodiscard]] bool startsWith(std::string_view s, std::string_view prefix) +{ + return s.size() >= prefix.size() && s.substr(0, prefix.size()) == prefix; +} + +[[nodiscard]] std::string normalizeUrlPrefix(std::string prefix) +{ + if (prefix.empty()) + prefix = "/"; + if (prefix.front() != '/') + prefix.insert(prefix.begin(), '/'); + if (prefix.size() > 1 && prefix.back() == '/') + prefix.pop_back(); + return prefix; +} + +[[nodiscard]] std::string_view guessMimeType(std::filesystem::path const& filePath) +{ + auto ext = filePath.extension().string(); + std::ranges::transform(ext, ext.begin(), [](unsigned char c) { return (char)std::tolower(c); }); + + if (ext == ".html" || ext == ".htm") + return "text/html"; + if (ext == ".css") + return "text/css"; + if (ext == ".js") + return "application/javascript"; + if (ext == ".json") + return "application/json"; + if (ext == ".svg") + return "image/svg+xml"; + if (ext == ".png") + return "image/png"; + if (ext == ".jpg" || ext == ".jpeg") + return "image/jpeg"; + if (ext == ".ico") + return "image/x-icon"; + if (ext == ".woff2") + return "font/woff2"; + if (ext == ".woff") + return "font/woff"; + if (ext == ".ttf") + return "font/ttf"; + if (ext == ".txt") + return "text/plain"; + + return "application/octet-stream"; +} + +[[nodiscard]] std::optional resolveStaticFile( + std::vector const& mounts, + std::string_view urlPath) +{ + if (mounts.empty()) + return std::nullopt; + if (!startsWith(urlPath, "/")) + return std::nullopt; + + // Longest-prefix match. + MountPoint const* best = nullptr; + for (auto const& m : mounts) { + if (startsWith(urlPath, m.urlPrefix) && (!best || m.urlPrefix.size() > best->urlPrefix.size())) + best = &m; + } + if (!best) + return std::nullopt; + + std::string_view remainder = urlPath.substr(best->urlPrefix.size()); + if (!remainder.empty() && remainder.front() == '/') + remainder.remove_prefix(1); + + std::filesystem::path relativePath = std::filesystem::path(std::string(remainder)).lexically_normal(); + if (relativePath.empty() || urlPath.back() == '/') + relativePath /= "index.html"; + + // Basic path traversal protection: reject any ".." segments. + for (auto const& part : relativePath) { + if (part == "..") + return std::nullopt; + } + + std::filesystem::path candidate = (best->fsRoot / relativePath).lexically_normal(); + return candidate; +} + +} // namespace + struct HttpServer::Impl { - httplib::Server server_; std::thread serverThread_; + std::atomic_bool running_{false}; + + std::mutex startMutex_; + std::condition_variable startCv_; + bool startNotified_ = false; + std::string startError_; + uint16_t port_ = 0; - bool setupWasCalled_ = false; bool printPortToStdout_ = false; + std::mutex mountsMutex_; + std::vector mounts_; + + uWS::Loop* loop_ = nullptr; + us_listen_socket_t* listenSocket_ = nullptr; + static void handleSignal(int) { // Temporarily holds the current active HttpServer @@ -35,69 +154,174 @@ struct HttpServer::Impl } } } + + void notifyStart(std::string errorMessage = {}) + { + std::lock_guard lock(startMutex_); + startError_ = std::move(errorMessage); + startNotified_ = true; + startCv_.notify_one(); + } }; HttpServer::HttpServer() : impl_(new Impl()) {} -HttpServer::~HttpServer() { +HttpServer::~HttpServer() +{ if (isRunning()) stop(); } -void HttpServer::go( - std::string const& interfaceAddr, - uint16_t port, - uint32_t waitMs) +void HttpServer::go(std::string const& interfaceAddr, uint16_t port, uint32_t waitMs) { - if (!impl_->setupWasCalled_) { - // Allow derived class to set up the server - setup(impl_->server_); - impl_->setupWasCalled_ = true; - } - - if (impl_->server_.is_running() || impl_->serverThread_.joinable()) + if (impl_->running_ || impl_->serverThread_.joinable()) raise("HttpServer is already running"); - if (port == 0) { - impl_->port_ = impl_->server_.bind_to_any_port(interfaceAddr); - } - else { - impl_->port_ = port; - impl_->server_.bind_to_port(interfaceAddr, port); + // Reset start state. + { + std::lock_guard lock(impl_->startMutex_); + impl_->startNotified_ = false; + impl_->startError_.clear(); } impl_->serverThread_ = std::thread( - [this, interfaceAddr] + [this, interfaceAddr, port] { - if (impl_->printPortToStdout_) - std::cout << "====== Running on port " << impl_->port_ << " ======" << std::endl; - else - log().info("====== Running on port {} ======", impl_->port_); - impl_->server_.listen_after_bind(); + try { + uWS::App app; + + // Allow derived class to set up the server + setup(app); + + // Copy mounts to avoid locking in the hot path. + std::vector mountsCopy; + { + std::lock_guard lock(impl_->mountsMutex_); + mountsCopy = impl_->mounts_; + } + + if (!mountsCopy.empty()) { + app.get( + "/*", + [mounts = std::move(mountsCopy)](auto* res, auto* req) mutable + { + auto urlPath = req->getUrl(); + auto candidate = resolveStaticFile(mounts, urlPath); + if (!candidate || !std::filesystem::exists(*candidate) || + !std::filesystem::is_regular_file(*candidate)) { + res->writeStatus("404 Not Found"); + res->writeHeader("Content-Type", "text/plain"); + res->end("Not found"); + return; + } + + std::ifstream ifs(*candidate, std::ios::binary); + if (!ifs) { + res->writeStatus("500 Internal Server Error"); + res->writeHeader("Content-Type", "text/plain"); + res->end("Failed to open file"); + return; + } + + std::string content; + ifs.seekg(0, std::ios::end); + content.resize(static_cast(ifs.tellg())); + ifs.seekg(0, std::ios::beg); + if (!content.empty()) { + ifs.read(content.data(), static_cast(content.size())); + } + + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", guessMimeType(*candidate)); + res->end(content); + }); + } + + app.listen( + interfaceAddr, + port, + [this, interfaceAddr, port](us_listen_socket_t* listenSocket) + { + if (!listenSocket) { + impl_->notifyStart( + fmt::format("Could not start HttpServer on {}:{}", interfaceAddr, port)); + return; + } + + impl_->listenSocket_ = listenSocket; + impl_->loop_ = uWS::Loop::get(); + + // Determine actual port (port may be 0 for ephemeral). + impl_->port_ = static_cast( + us_socket_local_port(0, reinterpret_cast(listenSocket))); + + impl_->running_ = true; + impl_->notifyStart(); + + if (impl_->printPortToStdout_) + std::cout << "====== Running on port " << impl_->port_ << " ======" << std::endl; + else + log().info("====== Running on port {} ======", impl_->port_); + }); + + // If listen failed, exit without running the loop. + if (!impl_->running_) { + if (!impl_->startNotified_) { + impl_->notifyStart(fmt::format("Could not start HttpServer on {}:{}", interfaceAddr, port)); + } + return; + } + + app.run(); + } + catch (std::exception const& e) { + impl_->notifyStart(e.what()); + } + + impl_->running_ = false; + impl_->listenSocket_ = nullptr; + impl_->loop_ = nullptr; }); - std::this_thread::sleep_for(std::chrono::milliseconds(waitMs)); - if (!impl_->server_.is_running() || !impl_->server_.is_valid()) - raise(fmt::format("Could not start HttpServer on {}:{}", interfaceAddr, port)); + std::unique_lock lk(impl_->startMutex_); + if (!impl_->startCv_.wait_for( + lk, + std::chrono::milliseconds(waitMs), + [this] { return impl_->startNotified_; })) { + raise(fmt::format("Could not start HttpServer on {}:{} (timeout)", interfaceAddr, port)); + } + + if (!impl_->startError_.empty()) + raise(impl_->startError_); } -bool HttpServer::isRunning() { - return impl_->server_.is_running(); +bool HttpServer::isRunning() +{ + return impl_->running_; } -void HttpServer::stop() { - if (!impl_->server_.is_running()) +void HttpServer::stop() +{ + if (!impl_->serverThread_.joinable()) return; - impl_->server_.stop(); - impl_->serverThread_.join(); + if (impl_->loop_ && impl_->listenSocket_) { + auto* loop = impl_->loop_; + auto* listenSocket = impl_->listenSocket_; + loop->defer([listenSocket]() { us_listen_socket_close(0, listenSocket); }); + } + + if (impl_->serverThread_.get_id() != std::this_thread::get_id()) + impl_->serverThread_.join(); } -uint16_t HttpServer::port() const { +uint16_t HttpServer::port() const +{ return impl_->port_; } -void HttpServer::waitForSignal() { +void HttpServer::waitForSignal() +{ // So the signal handler knows what to call activeHttpServer = this; @@ -107,24 +331,43 @@ void HttpServer::waitForSignal() { // Wait for the signal handler to stop us, or the server to shut down on its own. while (isRunning()) { - std::this_thread::sleep_for(std::chrono::milliseconds (200)); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); } activeHttpServer = nullptr; } -bool HttpServer::mountFileSystem(const std::string& pathFromTo) +bool HttpServer::mountFileSystem(std::string const& pathFromTo) { using namespace std::ranges; - auto parts = pathFromTo | views::split(':') | views::transform([](auto&& s){return std::string(&*s.begin(), distance(s));}); + auto parts = pathFromTo | views::split(':') | + views::transform([](auto&& s) { return std::string(&*s.begin(), distance(s)); }); auto partsVec = std::vector(parts.begin(), parts.end()); - if (partsVec.size() == 1) - return impl_->server_.set_mount_point("/", partsVec[0]); - return impl_->server_.set_mount_point(partsVec[0], partsVec[1]); + std::string urlPrefix; + std::filesystem::path fsRoot; + if (partsVec.size() == 1) { + urlPrefix = "/"; + fsRoot = partsVec[0]; + } else if (partsVec.size() == 2) { + urlPrefix = partsVec[0]; + fsRoot = partsVec[1]; + } else { + return false; + } + + urlPrefix = normalizeUrlPrefix(std::move(urlPrefix)); + + if (!std::filesystem::exists(fsRoot) || !std::filesystem::is_directory(fsRoot)) + return false; + + std::lock_guard lock(impl_->mountsMutex_); + impl_->mounts_.push_back(MountPoint{std::move(urlPrefix), std::move(fsRoot)}); + return true; } -void HttpServer::printPortToStdOut(bool enabled) { +void HttpServer::printPortToStdOut(bool enabled) +{ impl_->printPortToStdout_ = enabled; } diff --git a/libs/http-service/CMakeLists.txt b/libs/http-service/CMakeLists.txt index ce3d733e..23c2f847 100644 --- a/libs/http-service/CMakeLists.txt +++ b/libs/http-service/CMakeLists.txt @@ -18,6 +18,7 @@ target_include_directories(mapget-http-service target_link_libraries(mapget-http-service PUBLIC httplib::httplib + uWebSockets yaml-cpp CLI11::CLI11 nlohmann_json_schema_validator diff --git a/libs/http-service/include/mapget/http-service/http-client.h b/libs/http-service/include/mapget/http-service/http-client.h index 67c293d6..7fad612d 100644 --- a/libs/http-service/include/mapget/http-service/http-client.h +++ b/libs/http-service/include/mapget/http-service/http-client.h @@ -18,7 +18,7 @@ class HttpClient * endpoint, and caches the result for the lifetime of this object. * @param enableCompression Enable gzip compression for responses (default: true) */ - explicit HttpClient(std::string const& host, uint16_t port, httplib::Headers headers = {}, bool enableCompression = true); + explicit HttpClient(std::string const& host, uint16_t port, AuthHeaders headers = {}, bool enableCompression = true); ~HttpClient(); /** diff --git a/libs/http-service/include/mapget/http-service/http-service.h b/libs/http-service/include/mapget/http-service/http-service.h index 41f834c8..1e87225d 100644 --- a/libs/http-service/include/mapget/http-service/http-service.h +++ b/libs/http-service/include/mapget/http-service/http-service.h @@ -1,6 +1,5 @@ #pragma once -#include "httplib.h" #include "mapget/detail/http-server.h" #include "mapget/model/featurelayer.h" #include "mapget/model/stream.h" @@ -56,7 +55,7 @@ class HttpService : public HttpServer, public Service ~HttpService() override; protected: - void setup(httplib::Server& server) override; + void setup(uWS::App& app) override; private: struct Impl; diff --git a/libs/http-service/src/http-client.cpp b/libs/http-service/src/http-client.cpp index a002c34f..95ebdbea 100644 --- a/libs/http-service/src/http-client.cpp +++ b/libs/http-service/src/http-client.cpp @@ -11,10 +11,13 @@ struct HttpClient::Impl { std::shared_ptr stringPoolProvider_; httplib::Headers headers_; - Impl(std::string const& host, uint16_t port, httplib::Headers headers, bool enableCompression) : + Impl(std::string const& host, uint16_t port, AuthHeaders headers, bool enableCompression) : client_(host, port), - headers_(std::move(headers)) + headers_() { + for (auto const& [k, v] : headers) { + headers_.emplace(k, v); + } // Add Accept-Encoding header if compression is enabled and not already present if (enableCompression) { bool hasAcceptEncoding = false; @@ -51,7 +54,7 @@ struct HttpClient::Impl { } }; -HttpClient::HttpClient(const std::string& host, uint16_t port, httplib::Headers headers, bool enableCompression) : impl_( +HttpClient::HttpClient(const std::string& host, uint16_t port, AuthHeaders headers, bool enableCompression) : impl_( std::make_unique(host, port, std::move(headers), enableCompression)) {} HttpClient::~HttpClient() = default; diff --git a/libs/http-service/src/http-service.cpp b/libs/http-service/src/http-service.cpp index 0137a1b7..9f09586f 100644 --- a/libs/http-service/src/http-service.cpp +++ b/libs/http-service/src/http-service.cpp @@ -1,19 +1,29 @@ #include "http-service.h" + +#include "cli.h" #include "mapget/log.h" #include "mapget/service/config.h" +#include + +#include #include -#include +#include #include #include #include +#include #include +#include +#include +#include +#include #include -#include "cli.h" -#include "httplib.h" + #include "nlohmann/json-schema.hpp" #include "nlohmann/json.hpp" #include "yaml-cpp/yaml.h" + #include #ifdef __linux__ @@ -27,27 +37,28 @@ namespace { /** - * Simple gzip compressor for streaming compression + * Simple gzip compressor for streaming compression. */ -class GzipCompressor { +class GzipCompressor +{ public: - GzipCompressor() { + GzipCompressor() + { strm_.zalloc = Z_NULL; strm_.zfree = Z_NULL; strm_.opaque = Z_NULL; // 16+MAX_WBITS enables gzip format (not just deflate) - int ret = deflateInit2(&strm_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, - 16 + MAX_WBITS, 8, Z_DEFAULT_STRATEGY); + int ret = deflateInit2( + &strm_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 16 + MAX_WBITS, 8, Z_DEFAULT_STRATEGY); if (ret != Z_OK) { throw std::runtime_error("Failed to initialize gzip compressor"); } } - ~GzipCompressor() { - deflateEnd(&strm_); - } + ~GzipCompressor() { deflateEnd(&strm_); } - std::string compress(const char* data, size_t size, int flush_mode = Z_NO_FLUSH) { + std::string compress(const char* data, size_t size, int flush_mode = Z_NO_FLUSH) + { std::string result; if (size == 0 && flush_mode == Z_NO_FLUSH) { return result; @@ -60,12 +71,12 @@ class GzipCompressor { do { strm_.avail_out = sizeof(outbuf); strm_.next_out = reinterpret_cast(outbuf); - + int ret = deflate(&strm_, flush_mode); if (ret == Z_STREAM_ERROR) { throw std::runtime_error("Gzip compression failed"); } - + size_t have = sizeof(outbuf) - strm_.avail_out; result.append(outbuf, have); } while (strm_.avail_out == 0); @@ -73,19 +84,26 @@ class GzipCompressor { return result; } - std::string finish() { - return compress(nullptr, 0, Z_FINISH); - } + std::string finish() { return compress(nullptr, 0, Z_FINISH); } private: z_stream strm_{}; }; -/** - * Recursively convert a YAML node to a JSON object, - * with special handling for sensitive fields. - * The function returns a nlohmann::json object and updates maskedSecretMap. - */ +[[nodiscard]] AuthHeaders authHeadersFromRequest(uWS::HttpRequest* req) +{ + AuthHeaders headers; + for (auto const& [k, v] : *req) { + headers.emplace(std::string(k), std::string(v)); + } + return headers; +} + +[[nodiscard]] bool containsGzip(std::string_view acceptEncoding) +{ + return !acceptEncoding.empty() && acceptEncoding.find("gzip") != std::string_view::npos; +} + } // namespace struct HttpService::Impl @@ -95,64 +113,47 @@ struct HttpService::Impl mutable std::atomic binaryRequestCounter_{0}; mutable std::atomic jsonRequestCounter_{0}; - explicit Impl(HttpService& self, const HttpServiceConfig& config) - : self_(self), config_(config) {} + explicit Impl(HttpService& self, const HttpServiceConfig& config) : self_(self), config_(config) {} - enum class ResponseType { - Binary, - Json - }; + enum class ResponseType { Binary, Json }; + + void tryMemoryTrim(ResponseType responseType) const + { + uint64_t interval = + (responseType == ResponseType::Binary) ? config_.memoryTrimIntervalBinary : config_.memoryTrimIntervalJson; + + if (interval == 0) { + return; + } + + auto& counter = (responseType == ResponseType::Binary) ? binaryRequestCounter_ : jsonRequestCounter_; + auto count = counter.fetch_add(1, std::memory_order_relaxed); + if ((count % interval) != 0) { + return; + } - void tryMemoryTrim(ResponseType responseType) const { - uint64_t interval = (responseType == ResponseType::Binary) - ? config_.memoryTrimIntervalBinary - : config_.memoryTrimIntervalJson; - - if (interval > 0) { - auto& counter = (responseType == ResponseType::Binary) - ? binaryRequestCounter_ - : jsonRequestCounter_; - - auto count = counter.fetch_add(1, std::memory_order_relaxed); - if ((count % interval) == 0) { #ifdef __linux__ - // Only log in debug builds to reduce overhead - #ifndef NDEBUG - const char* typeStr = (responseType == ResponseType::Binary) ? "binary" : "JSON"; - log().debug("Trimming memory after {} {} requests (interval: {})", count, typeStr, interval); - #endif - malloc_trim(0); +#ifndef NDEBUG + const char* typeStr = (responseType == ResponseType::Binary) ? "binary" : "JSON"; + log().debug("Trimming memory after {} {} requests (interval: {})", count, typeStr, interval); +#endif + malloc_trim(0); #endif - // On non-Linux platforms, this is a no-op but we still track the counter - } - } } - // Use a shared buffer for the responses and a mutex for thread safety. - struct HttpTilesRequestState + struct TilesStreamState : std::enable_shared_from_this { static constexpr auto binaryMimeType = "application/binary"; static constexpr auto jsonlMimeType = "application/jsonl"; static constexpr auto anyMimeType = "*/*"; - std::mutex mutex_; - std::condition_variable resultEvent_; - - uint64_t requestId_; - std::stringstream buffer_; - std::string responseType_; - std::unique_ptr writer_; - std::vector requests_; - TileLayerStream::StringPoolOffsetMap stringOffsets_; - std::unique_ptr compressor_; // Store compressor per request - - HttpTilesRequestState() + explicit TilesStreamState(Impl const& impl, uWS::HttpResponse* res, uWS::Loop* loop) + : impl_(impl), res_(res), loop_(loop) { static std::atomic_uint64_t nextRequestId; - writer_ = std::make_unique( - [&, this](auto&& msg, auto&& msgType) { buffer_ << msg; }, - stringOffsets_); requestId_ = nextRequestId++; + writer_ = std::make_unique( + [this](auto&& msg, auto&& /*msgType*/) { appendOutgoingUnlocked(msg); }, stringOffsets_); } void parseRequestFromJson(nlohmann::json const& requestJson) @@ -161,57 +162,194 @@ struct HttpService::Impl std::string layerId = requestJson["layerId"]; std::vector tileIds; tileIds.reserve(requestJson["tileIds"].size()); - for (auto const& tid : requestJson["tileIds"].get>()) + for (auto const& tid : requestJson["tileIds"].get>()) { tileIds.emplace_back(tid); - requests_ - .push_back(std::make_shared(mapId, layerId, std::move(tileIds))); + } + requests_.push_back(std::make_shared(mapId, layerId, std::move(tileIds))); } - void setResponseType(std::string const& s) + [[nodiscard]] bool setResponseTypeFromAccept(std::string_view acceptHeader, std::string& error) { - responseType_ = s; - if (responseType_ == HttpTilesRequestState::binaryMimeType) - return; - if (responseType_ == HttpTilesRequestState::jsonlMimeType) - return; - if (responseType_ == HttpTilesRequestState::anyMimeType) { + responseType_ = std::string(acceptHeader); + if (responseType_.empty()) + responseType_ = anyMimeType; + if (responseType_ == anyMimeType) responseType_ = binaryMimeType; + + if (responseType_ == binaryMimeType) { + trimResponseType_ = ResponseType::Binary; + return true; + } + if (responseType_ == jsonlMimeType) { + trimResponseType_ = ResponseType::Json; + return true; + } + + error = "Unknown Accept header value: " + responseType_; + return false; + } + + void enableGzip() { compressor_ = std::make_unique(); } + + void onAborted() + { + if (aborted_.exchange(true)) return; + for (auto const& req : requests_) { + if (!req->isDone()) { + impl_.self_.abort(req); + } } - raise(fmt::format("Unknown Accept-Header value {}", responseType_)); } void addResult(TileLayer::Ptr const& result) { - std::unique_lock lock(mutex_); - log().debug("Response ready: {}", MapTileKey(*result).toString()); - if (responseType_ == binaryMimeType) { - // Binary response - writer_->write(result); - } - else { - // JSON response - optimize with compact dump settings - // TODO: Implement direct streaming with result->writeGeoJsonTo(buffer_) - // to avoid intermediate JSON object creation entirely - auto json = result->toJson(); - // Use compact dump: no indentation, no spaces, ignore errors - // This reduces string allocation overhead - buffer_ << json.dump(-1, ' ', false, nlohmann::json::error_handler_t::ignore) << "\n"; - } - resultEvent_.notify_one(); + { + std::lock_guard lock(mutex_); + if (aborted_) + return; + + log().debug("Response ready: {}", MapTileKey(*result).toString()); + if (responseType_ == binaryMimeType) { + writer_->write(result); + } else { + auto dumped = result->toJson().dump( + -1, ' ', false, nlohmann::json::error_handler_t::ignore); + appendOutgoingUnlocked(dumped); + appendOutgoingUnlocked("\n"); + } + } + scheduleDrain(); + } + + void onRequestDone() + { + { + std::lock_guard lock(mutex_); + if (aborted_) + return; + + bool allDoneNow = std::all_of( + requests_.begin(), requests_.end(), [](auto const& r) { return r->isDone(); }); + + if (allDoneNow && !allDone_) { + allDone_ = true; + if (responseType_ == binaryMimeType && !endOfStreamSent_) { + writer_->sendEndOfStream(); + endOfStreamSent_ = true; + } + } + } + scheduleDrain(); + } + + void scheduleDrain() + { + if (aborted_ || responseEnded_) + return; + if (drainScheduled_.exchange(true)) + return; + + auto weak = weak_from_this(); + loop_->defer([weak = std::move(weak)]() mutable { + if (auto self = weak.lock()) { + self->drainOnLoop(); + } + }); + } + + void drainOnLoop() + { + drainScheduled_ = false; + if (aborted_ || responseEnded_) + return; + + constexpr size_t maxChunk = 64 * 1024; + + for (;;) { + std::string chunk; + bool done = false; + { + std::lock_guard lock(mutex_); + if (!pending_.empty()) { + size_t n = std::min(pending_.size(), maxChunk); + chunk.assign(pending_.data(), n); + pending_.erase(0, n); + } else { + if (allDone_ && compressor_ && !compressionFinished_) { + pending_.append(compressor_->finish()); + compressionFinished_ = true; + continue; + } + done = allDone_; + } + } + + if (!chunk.empty()) { + bool ok = res_->write(chunk); + if (!ok) { + // Backpressure: resume in onWritable. + return; + } + continue; + } + + if (done) { + responseEnded_ = true; + res_->end(); + impl_.tryMemoryTrim(trimResponseType_); + } + return; + } } + + void appendOutgoingUnlocked(std::string_view bytes) + { + if (bytes.empty()) + return; + + if (compressor_) { + pending_.append(compressor_->compress(bytes.data(), bytes.size())); + } else { + pending_.append(bytes); + } + } + + Impl const& impl_; + uWS::HttpResponse* res_; + uWS::Loop* loop_; + + std::mutex mutex_; + uint64_t requestId_ = 0; + + std::string responseType_; + ResponseType trimResponseType_ = ResponseType::Binary; + + std::string pending_; + std::unique_ptr writer_; + std::vector requests_; + TileLayerStream::StringPoolOffsetMap stringOffsets_; + + std::unique_ptr compressor_; + bool compressionFinished_ = false; + bool endOfStreamSent_ = false; + bool allDone_ = false; + + std::atomic_bool aborted_{false}; + std::atomic_bool drainScheduled_{false}; + std::atomic_bool responseEnded_{false}; }; mutable std::mutex clientRequestMapMutex_; - mutable std::unordered_map> requestStatePerClientId_; + mutable std::unordered_map> requestStatePerClientId_; - void abortRequestsForClientId(std::string clientId, std::shared_ptr newState = nullptr) const + void abortRequestsForClientId( + std::string const& clientId, + std::shared_ptr newState = nullptr) const { std::unique_lock clientRequestMapAccess(clientRequestMapMutex_); auto clientRequestIt = requestStatePerClientId_.find(clientId); if (clientRequestIt != requestStatePerClientId_.end()) { - // Ensure that any previous requests from the same clientId - // are finished post-haste! bool anySoftAbort = false; for (auto const& req : clientRequestIt->second->requests_) { if (!req->isDone()) { @@ -224,205 +362,173 @@ struct HttpService::Impl requestStatePerClientId_.erase(clientRequestIt); } if (newState) { - requestStatePerClientId_.emplace(clientId, newState); + requestStatePerClientId_.emplace(clientId, std::move(newState)); } } - /** - * Wraps around the generic mapget service's request() function - * to include httplib request decoding and response encoding. - */ - void handleTilesRequest(const httplib::Request& req, httplib::Response& res) const + void handleTilesRequest(uWS::HttpResponse* res, uWS::HttpRequest* req) const { - // Parse the JSON request. - nlohmann::json j = nlohmann::json::parse(req.body); - auto requestsJson = j["requests"]; - - // TODO: Limit number of requests to avoid DoS to other users. - // Within one HTTP request, all requested tiles from the same map+layer - // combination should be in a single LayerTilesRequest. - auto state = std::make_shared(); - log().info("Processing tiles request {}", state->requestId_); - for (auto& requestJson : requestsJson) { - state->parseRequestFromJson(requestJson); - } + auto* loop = uWS::Loop::get(); + auto state = std::make_shared(*this, res, loop); + + std::string accept = std::string(req->getHeader("accept")); + std::string acceptEncoding = std::string(req->getHeader("accept-encoding")); + auto clientHeaders = authHeadersFromRequest(req); + + res->onAborted([state]() { state->onAborted(); }); + + res->onData([this, + res, + state, + clientHeaders = std::move(clientHeaders), + accept = std::move(accept), + acceptEncoding = std::move(acceptEncoding), + body = std::string()](std::string_view chunk, bool last) mutable { + if (state->aborted_ || state->responseEnded_) + return; + + body.append(chunk.data(), chunk.size()); + if (!last) + return; - // Parse stringPoolOffsets. - if (j.contains("stringPoolOffsets")) { - for (auto& item : j["stringPoolOffsets"].items()) { - state->stringOffsets_[item.key()] = item.value().get(); + nlohmann::json j; + try { + j = nlohmann::json::parse(body); + } + catch (const std::exception& e) { + state->responseEnded_ = true; + res->writeStatus("400 Bad Request"); + res->writeHeader("Content-Type", "text/plain"); + res->end(std::string("Invalid JSON: ") + e.what()); + return; } - } - // Determine response type. - state->setResponseType(req.get_header_value("Accept")); + auto requestsIt = j.find("requests"); + if (requestsIt == j.end() || !requestsIt->is_array()) { + state->responseEnded_ = true; + res->writeStatus("400 Bad Request"); + res->writeHeader("Content-Type", "text/plain"); + res->end("Missing or invalid 'requests' array"); + return; + } - // Process requests. - for (auto& request : state->requests_) { - request->onFeatureLayer([state](auto&& layer) { state->addResult(layer); }); - request->onSourceDataLayer([state](auto&& layer) { state->addResult(layer); }); - request->onDone_ = [state](RequestStatus r) - { - state->resultEvent_.notify_one(); - }; - } - auto canProcess = self_.request( - state->requests_, - AuthHeaders{req.headers.begin(), req.headers.end()}); - - if (!canProcess) { - // Send a status report detailing for each request - // whether its data source is unavailable or it was aborted. - res.status = 400; - std::vector> requestStatuses{}; - for (const auto& r : state->requests_) { - requestStatuses.push_back(static_cast>(r->getStatus())); - if (r->getStatus() == RequestStatus::Unauthorized) { - res.status = 403; // Forbidden. - } + log().info("Processing tiles request {}", state->requestId_); + for (auto& requestJson : *requestsIt) { + state->parseRequestFromJson(requestJson); } - res.set_content( - nlohmann::json::object({{"requestStatuses", requestStatuses}}).dump(), - "application/json"); - return; - } - // Parse/Process clientId. - if (j.contains("clientId")) { - auto clientId = j["clientId"].get(); - abortRequestsForClientId(clientId, state); - } + if (j.contains("stringPoolOffsets")) { + for (auto& item : j["stringPoolOffsets"].items()) { + state->stringOffsets_[item.key()] = item.value().get(); + } + } - // Check if client accepts gzip compression - bool enableGzip = false; - if (req.has_header("Accept-Encoding")) { - std::string acceptEncoding = req.get_header_value("Accept-Encoding"); - enableGzip = acceptEncoding.find("gzip") != std::string::npos; - log().debug("Accept-Encoding header: '{}', enableGzip: {}", acceptEncoding, enableGzip); - } else { - log().debug("No Accept-Encoding header present"); - } + std::string acceptError; + if (!state->setResponseTypeFromAccept(accept, acceptError)) { + state->responseEnded_ = true; + res->writeStatus("400 Bad Request"); + res->writeHeader("Content-Type", "text/plain"); + res->end(acceptError); + return; + } - // Set Content-Encoding header if compression is enabled - if (enableGzip) { - res.set_header("Content-Encoding", "gzip"); - state->compressor_ = std::make_unique(); - log().debug("Set Content-Encoding: gzip header"); - } + const bool gzip = containsGzip(acceptEncoding); + if (gzip) { + state->enableGzip(); + } - // For efficiency, set up httplib to stream tile layer responses to client: - // (1) Lambda continuously supplies response data to httplib's DataSink, - // picking up data from state->buffer_ until all tile requests are done. - // Then, signal sink->done() to close the stream with a 200 status. - // Using chunked transfer encoding with optional manual compression. - // (2) Lambda acts as a cleanup routine, triggered by httplib upon request wrap-up. - // The success flag indicates if wrap-up was due to sink->done() or external factors - // like network errors or request aborts in lengthy tile requests (e.g., map-viewer). - res.set_chunked_content_provider( - state->responseType_, - [state](size_t offset, httplib::DataSink& sink) - { - std::unique_lock lock(state->mutex_); - - // Wait until there is data to be read. - std::string strBuf; - bool allDone = false; - state->resultEvent_.wait( - lock, - [&] - { - allDone = std::all_of( - state->requests_.begin(), - state->requests_.end(), - [](const auto& r) { return r->isDone(); }); - if (allDone && state->responseType_ == HttpTilesRequestState::binaryMimeType) - state->writer_->sendEndOfStream(); - strBuf = state->buffer_.str(); - return !strBuf.empty() || allDone; - }); + for (auto& request : state->requests_) { + request->onFeatureLayer([state](auto&& layer) { state->addResult(layer); }); + request->onSourceDataLayer([state](auto&& layer) { state->addResult(layer); }); + request->onDone_ = [state](RequestStatus) { state->onRequestDone(); }; + } - if (!strBuf.empty()) { - // Compress data if gzip is enabled - if (state->compressor_) { - std::string compressed = state->compressor_->compress(strBuf.data(), strBuf.size()); - if (!compressed.empty()) { - log().debug("Compressing: {} bytes -> {} bytes (request {})", - strBuf.size(), compressed.size(), state->requestId_); - sink.write(compressed.data(), compressed.size()); - } - } else { - log().debug("Streaming {} bytes (no compression)...", strBuf.size()); - sink.write(strBuf.data(), strBuf.size()); - } - sink.os.flush(); - state->buffer_.str(""); // Clear buffer content - state->buffer_.clear(); // Clear error flags - // Force release of internal buffer memory - std::stringstream().swap(state->buffer_); + auto canProcess = self_.request(state->requests_, clientHeaders); + if (!canProcess) { + state->responseEnded_ = true; + std::vector> requestStatuses{}; + bool anyUnauthorized = false; + for (auto const& r : state->requests_) { + auto status = r->getStatus(); + requestStatuses.emplace_back(static_cast>(status)); + anyUnauthorized |= (status == RequestStatus::Unauthorized); } + res->writeStatus(anyUnauthorized ? "403 Forbidden" : "400 Bad Request"); + res->writeHeader("Content-Type", "application/json"); + res->end(nlohmann::json::object({{"status", requestStatuses}}).dump()); + return; + } - // Call sink.done() when all requests are done. - if (allDone) { - // Finish compression if enabled - if (state->compressor_) { - std::string finalChunk = state->compressor_->finish(); - log().debug( - "Final compression chunk is {} bytes.", - strBuf.size(), finalChunk.size(), state->requestId_); - if (!finalChunk.empty()) { - sink.write(finalChunk.data(), finalChunk.size()); - } - } - sink.done(); - } + if (j.contains("clientId")) { + abortRequestsForClientId(j["clientId"].get(), state); + } - return true; - }, - // Network error/timeout of request to datasource: - // cleanup callback to abort the requests. - [state, this](bool success) - { - if (!success) { - log().warn("Aborting tiles request {}", state->requestId_); - for (auto& request : state->requests_) { - self_.abort(request); - } - } - else { - log().info("Tiles request {} was successful.", state->requestId_); - // Determine response type and trim accordingly - ResponseType respType = (state->responseType_ == HttpTilesRequestState::binaryMimeType) - ? ResponseType::Binary - : ResponseType::Json; - tryMemoryTrim(respType); - } + if (gzip) { + res->writeHeader("Content-Encoding", "gzip"); + } + + res->writeHeader("Content-Type", state->responseType_); + res->onWritable([state](uintmax_t) { + state->drainOnLoop(); + return !state->responseEnded_.load(); }); + + state->scheduleDrain(); + }); } - void handleAbortRequest(const httplib::Request& req, httplib::Response& res) const + void handleAbortRequest(uWS::HttpResponse* res) const { - // Parse the JSON request. - nlohmann::json j = nlohmann::json::parse(req.body); - if (j.contains("clientId")) { - auto const clientId = j["clientId"].get(); - abortRequestsForClientId(clientId); - } - else { - res.status = 400; - res.set_content("Missing clientId", "text/plain"); - } + auto aborted = std::make_shared(false); + res->onAborted([aborted]() { *aborted = true; }); + + res->onData([this, res, aborted, body = std::string()](std::string_view chunk, bool last) mutable { + if (*aborted) + return; + body.append(chunk.data(), chunk.size()); + if (!last) + return; + + try { + auto j = nlohmann::json::parse(body); + if (j.contains("clientId")) { + abortRequestsForClientId(j["clientId"].get()); + if (*aborted) + return; + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "text/plain"); + res->end("OK"); + return; + } + + if (*aborted) + return; + res->writeStatus("400 Bad Request"); + res->writeHeader("Content-Type", "text/plain"); + res->end("Missing clientId"); + } + catch (const std::exception& e) { + if (*aborted) + return; + res->writeStatus("400 Bad Request"); + res->writeHeader("Content-Type", "text/plain"); + res->end(std::string("Invalid JSON: ") + e.what()); + } + }); } - void handleSourcesRequest(const httplib::Request& req, httplib::Response& res) const + void handleSourcesRequest(uWS::HttpResponse* res, uWS::HttpRequest* req) const { auto sourcesInfo = nlohmann::json::array(); - for (auto& source : self_.info(AuthHeaders{req.headers.begin(), req.headers.end()})) { + for (auto& source : self_.info(authHeadersFromRequest(req))) { sourcesInfo.push_back(source.toJson()); } - res.set_content(sourcesInfo.dump(), "application/json"); + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "application/json"); + res->end(sourcesInfo.dump()); } - void handleStatusRequest(const httplib::Request&, httplib::Response& res) const + void handleStatusRequest(uWS::HttpResponse* res) const { auto serviceStats = self_.getStatistics(); auto cacheStats = self_.cache()->getStatistics(); @@ -430,74 +536,93 @@ struct HttpService::Impl std::ostringstream oss; oss << ""; oss << "

Status Information

"; - - // Output serviceStats oss << "

Service Statistics

"; - oss << "
" << serviceStats.dump(4) << "
"; // Indentation of 4 for pretty printing - - // Output cacheStats + oss << "
" << serviceStats.dump(4) << "
"; oss << "

Cache Statistics

"; - oss << "
" << cacheStats.dump(4) << "
"; // Indentation of 4 for pretty printing - + oss << "
" << cacheStats.dump(4) << "
"; oss << ""; - res.set_content(oss.str(), "text/html"); + + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "text/html"); + res->end(oss.str()); } - void handleLocateRequest(const httplib::Request& req, httplib::Response& res) const + void handleLocateRequest(uWS::HttpResponse* res) const { - // Parse the JSON request. - nlohmann::json j = nlohmann::json::parse(req.body); - auto requestsJson = j["requests"]; - auto allResponsesJson = nlohmann::json::array(); - - for (auto const& locateReqJson : requestsJson) { - LocateRequest locateReq{locateReqJson}; - auto responsesJson = nlohmann::json::array(); - for (auto const& resp : self_.locate(locateReq)) - responsesJson.emplace_back(resp.serialize()); - allResponsesJson.emplace_back(responsesJson); - } + auto aborted = std::make_shared(false); + res->onAborted([aborted]() { *aborted = true; }); - res.set_content( - nlohmann::json::object({{"responses", allResponsesJson}}).dump(), - "application/json"); + res->onData([this, res, aborted, body = std::string()](std::string_view chunk, bool last) mutable { + if (*aborted) + return; + body.append(chunk.data(), chunk.size()); + if (!last) + return; + + try { + nlohmann::json j = nlohmann::json::parse(body); + auto requestsJson = j["requests"]; + auto allResponsesJson = nlohmann::json::array(); + + for (auto const& locateReqJson : requestsJson) { + LocateRequest locateReq{locateReqJson}; + auto responsesJson = nlohmann::json::array(); + for (auto const& resp : self_.locate(locateReq)) + responsesJson.emplace_back(resp.serialize()); + allResponsesJson.emplace_back(responsesJson); + } + + if (*aborted) + return; + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "application/json"); + res->end(nlohmann::json::object({{"responses", allResponsesJson}}).dump()); + } + catch (const std::exception& e) { + if (*aborted) + return; + res->writeStatus("400 Bad Request"); + res->writeHeader("Content-Type", "text/plain"); + res->end(std::string("Invalid JSON: ") + e.what()); + } + }); } - static bool openConfigFile(std::ifstream& configFile, httplib::Response& res) + static bool openConfigFile(std::ifstream& configFile, uWS::HttpResponse* res) { auto configFilePath = DataSourceConfigService::get().getConfigFilePath(); if (!configFilePath.has_value()) { - res.status = 404; // Not found. - res.set_content( - "The config file path is not set. Check the server configuration.", - "text/plain"); + res->writeStatus("404 Not Found"); + res->writeHeader("Content-Type", "text/plain"); + res->end("The config file path is not set. Check the server configuration."); return false; } std::filesystem::path path = *configFilePath; - if (!configFilePath || !std::filesystem::exists(path)) { - res.status = 404; // Not found. - res.set_content("The server does not have a config file.", "text/plain"); + if (!std::filesystem::exists(path)) { + res->writeStatus("404 Not Found"); + res->writeHeader("Content-Type", "text/plain"); + res->end("The server does not have a config file."); return false; } configFile.open(*configFilePath); if (!configFile) { - res.status = 500; // Internal Server Error. - res.set_content("Failed to open config file.", "text/plain"); + res->writeStatus("500 Internal Server Error"); + res->writeHeader("Content-Type", "text/plain"); + res->end("Failed to open config file."); return false; } return true; } - static void handleGetConfigRequest(const httplib::Request& req, httplib::Response& res) + static void handleGetConfigRequest(uWS::HttpResponse* res) { if (!isGetConfigEndpointEnabled()) { - res.status = 403; // Forbidden. - res.set_content( - "The GET /config endpoint is disabled by the server administrator.", - "text/plain"); + res->writeStatus("403 Forbidden"); + res->writeHeader("Content-Type", "text/plain"); + res->end("The GET /config endpoint is disabled by the server administrator."); return; } @@ -505,10 +630,10 @@ struct HttpService::Impl if (!openConfigFile(configFile, res)) { return; } + nlohmann::json jsonSchema = DataSourceConfigService::get().getDataSourceConfigSchema(); try { - // Load config YAML, expose the parts which clients may edit. YAML::Node configYaml = YAML::Load(configFile); nlohmann::json jsonConfig; std::unordered_map maskedSecretMap; @@ -522,147 +647,168 @@ struct HttpService::Impl combinedJson["model"] = jsonConfig; combinedJson["readOnly"] = !isPostConfigEndpointEnabled(); - // Set the response - res.status = 200; // OK - res.set_content(combinedJson.dump(2), "application/json"); + res->writeStatus("200 OK"); + res->writeHeader("Content-Type", "application/json"); + res->end(combinedJson.dump(2)); } catch (const std::exception& e) { - res.status = 500; // Internal Server Error - res.set_content("Error processing config file: " + std::string(e.what()), "text/plain"); + res->writeStatus("500 Internal Server Error"); + res->writeHeader("Content-Type", "text/plain"); + res->end(std::string("Error processing config file: ") + e.what()); } } - static void handlePostConfigRequest(const httplib::Request& req, httplib::Response& res) + void handlePostConfigRequest(uWS::HttpResponse* res) const { if (!isPostConfigEndpointEnabled()) { - res.status = 403; // Forbidden. - res.set_content( - "The POST /config endpoint is not enabled by the server administrator.", - "text/plain"); + res->writeStatus("403 Forbidden"); + res->writeHeader("Content-Type", "text/plain"); + res->end("The POST /config endpoint is not enabled by the server administrator."); return; } - std::mutex mtx; - std::condition_variable cv; - bool update_done = false; - - std::ifstream configFile; - if (!openConfigFile(configFile, res)) { - return; - } + struct ConfigUpdateState : std::enable_shared_from_this + { + uWS::HttpResponse* res = nullptr; + uWS::Loop* loop = nullptr; + std::atomic_bool aborted{false}; + std::atomic_bool done{false}; + std::atomic_bool wroteConfig{false}; + std::unique_ptr subscription; + std::string body; + }; + + auto state = std::make_shared(); + state->res = res; + state->loop = uWS::Loop::get(); + + res->onAborted([state]() { + state->aborted = true; + state->done = true; + state->subscription.reset(); + }); + + res->onData([state](std::string_view chunk, bool last) mutable { + if (state->aborted) + return; + state->body.append(chunk.data(), chunk.size()); + if (!last) + return; - // Subscribe to configuration changes. - auto subscription = DataSourceConfigService::get().subscribe( - [&](const std::vector& serviceConfigNodes) - { - std::lock_guard lock(mtx); - res.status = 200; - res.set_content("Configuration updated and applied successfully.", "text/plain"); - update_done = true; - cv.notify_one(); - }, - [&](const std::string& error) - { - std::lock_guard lock(mtx); - res.status = 500; - res.set_content("Error applying the configuration: " + error, "text/plain"); - update_done = true; - cv.notify_one(); - }); + std::ifstream configFile; + if (!Impl::openConfigFile(configFile, state->res)) { + state->done = true; + return; + } - // Parse the JSON from the request body. - nlohmann::json jsonConfig; - try { - jsonConfig = nlohmann::json::parse(req.body); - } - catch (const nlohmann::json::parse_error& e) { - res.status = 400; // Bad Request - res.set_content("Invalid JSON format: " + std::string(e.what()), "text/plain"); - return; - } + nlohmann::json jsonConfig; + try { + jsonConfig = nlohmann::json::parse(state->body); + } + catch (const nlohmann::json::parse_error& e) { + state->res->writeStatus("400 Bad Request"); + state->res->writeHeader("Content-Type", "text/plain"); + state->res->end(std::string("Invalid JSON format: ") + e.what()); + state->done = true; + return; + } - // Validate JSON against schema. - try { - DataSourceConfigService::get().validateDataSourceConfig(jsonConfig); - } - catch (const std::exception& e) { - res.status = 500; // Internal Server Error. - res.set_content("Validation failed: " + std::string(e.what()), "text/plain"); - return; - } + try { + DataSourceConfigService::get().validateDataSourceConfig(jsonConfig); + } + catch (const std::exception& e) { + state->res->writeStatus("500 Internal Server Error"); + state->res->writeHeader("Content-Type", "text/plain"); + state->res->end(std::string("Validation failed: ") + e.what()); + state->done = true; + return; + } - // Load the YAML, parse the secrets. - auto yamlConfig = YAML::Load(configFile); - std::unordered_map maskedSecrets; - yamlToJson(yamlConfig, true, &maskedSecrets); + auto yamlConfig = YAML::Load(configFile); + std::unordered_map maskedSecrets; + yamlToJson(yamlConfig, true, &maskedSecrets); - // Create YAML nodes from JSON nodes. - for (auto const& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { - if (jsonConfig.contains(key)) - yamlConfig[key] = jsonToYaml(jsonConfig[key], maskedSecrets); - } + for (auto const& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { + if (jsonConfig.contains(key)) + yamlConfig[key] = jsonToYaml(jsonConfig[key], maskedSecrets); + } - // Write the YAML to configFilePath. - update_done = false; - configFile.close(); - log().trace("Writing new config."); - std::ofstream newConfigFile(*DataSourceConfigService::get().getConfigFilePath()); - newConfigFile << yamlConfig; - newConfigFile.close(); - - // Wait for the subscription callback. - std::unique_lock lk(mtx); - if (!cv.wait_for(lk, std::chrono::seconds(60), [&] { return update_done; })) { - res.status = 500; // Internal Server Error. - res.set_content("Timeout while waiting for config to update.", "text/plain"); - } + // Subscribe before writing; ignore any callbacks that happen before we write. + state->subscription = DataSourceConfigService::get().subscribe( + [state](std::vector const&) mutable { + if (!state->wroteConfig) { + return; + } + if (state->done.exchange(true) || state->aborted) + return; + state->loop->defer([state]() mutable { + if (state->aborted) + return; + state->res->writeStatus("200 OK"); + state->res->writeHeader("Content-Type", "text/plain"); + state->res->end("Configuration updated and applied successfully."); + state->subscription.reset(); + }); + }, + [state](std::string const& error) mutable { + if (!state->wroteConfig) { + return; + } + if (state->done.exchange(true) || state->aborted) + return; + state->loop->defer([state, error]() mutable { + if (state->aborted) + return; + state->res->writeStatus("500 Internal Server Error"); + state->res->writeHeader("Content-Type", "text/plain"); + state->res->end(std::string("Error applying the configuration: ") + error); + state->subscription.reset(); + }); + }); + + configFile.close(); + log().trace("Writing new config."); + state->wroteConfig = true; + std::ofstream newConfigFile(*DataSourceConfigService::get().getConfigFilePath()); + newConfigFile << yamlConfig; + newConfigFile.close(); + + // Timeout fail-safe (rare endpoint; ok to spawn a thread). + std::thread([weak = state->weak_from_this()]() { + std::this_thread::sleep_for(std::chrono::seconds(60)); + if (auto state = weak.lock()) { + if (state->done.exchange(true) || state->aborted) + return; + state->loop->defer([state]() mutable { + if (state->aborted) + return; + state->res->writeStatus("500 Internal Server Error"); + state->res->writeHeader("Content-Type", "text/plain"); + state->res->end("Timeout while waiting for config to update."); + state->subscription.reset(); + }); + } + }).detach(); + }); } }; HttpService::HttpService(Cache::Ptr cache, const HttpServiceConfig& config) - : Service(std::move(cache), config.watchConfig, config.defaultTtl), - impl_(std::make_unique(*this, config)) + : Service(std::move(cache), config.watchConfig, config.defaultTtl), impl_(std::make_unique(*this, config)) { } HttpService::~HttpService() = default; -void HttpService::setup(httplib::Server& server) +void HttpService::setup(uWS::App& app) { - server.Post( - "/tiles", - [&](const httplib::Request& req, httplib::Response& res) - { impl_->handleTilesRequest(req, res); }); - - server.Post( - "/abort", - [&](const httplib::Request& req, httplib::Response& res) - { impl_->handleAbortRequest(req, res); }); - - server.Get( - "/sources", - [this](const httplib::Request& req, httplib::Response& res) - { impl_->handleSourcesRequest(req, res); }); - - server.Get( - "/status", - [this](const httplib::Request& req, httplib::Response& res) - { impl_->handleStatusRequest(req, res); }); - - server.Post( - "/locate", - [this](const httplib::Request& req, httplib::Response& res) - { impl_->handleLocateRequest(req, res); }); - - server.Get( - "/config", - [this](const httplib::Request& req, httplib::Response& res) - { impl_->handleGetConfigRequest(req, res); }); - - server.Post( - "/config", - [this](const httplib::Request& req, httplib::Response& res) - { impl_->handlePostConfigRequest(req, res); }); + app.post("/tiles", [this](auto* res, auto* req) { impl_->handleTilesRequest(res, req); }); + app.post("/abort", [this](auto* res, auto* /*req*/) { impl_->handleAbortRequest(res); }); + app.get("/sources", [this](auto* res, auto* req) { impl_->handleSourcesRequest(res, req); }); + app.get("/status", [this](auto* res, auto* /*req*/) { impl_->handleStatusRequest(res); }); + app.post("/locate", [this](auto* res, auto* /*req*/) { impl_->handleLocateRequest(res); }); + app.get("/config", [](auto* res, auto* /*req*/) { Impl::handleGetConfigRequest(res); }); + app.post("/config", [this](auto* res, auto* /*req*/) { impl_->handlePostConfigRequest(res); }); } } // namespace mapget diff --git a/libs/service/src/datasource.cpp b/libs/service/src/datasource.cpp index 2e15d0a6..8074fcd4 100644 --- a/libs/service/src/datasource.cpp +++ b/libs/service/src/datasource.cpp @@ -1,4 +1,6 @@ #include "datasource.h" +#include +#include #include #include #include @@ -54,6 +56,7 @@ TileLayer::Ptr DataSource::get(const MapTileKey& k, Cache::Ptr& cache, DataSourc void DataSource::requireAuthHeaderRegexMatchOption(std::string header, std::regex re) { + std::ranges::transform(header, header.begin(), [](unsigned char c) { return (char)std::tolower(c); }); authHeaderAlternatives_.insert({std::move(header), std::move(re)}); } @@ -64,7 +67,9 @@ bool DataSource::isDataSourceAuthorized( return true; for (auto const& [k, v] : clientHeaders) { - auto authHeaderPatternIt = authHeaderAlternatives_.find(k); + auto key = k; + std::ranges::transform(key, key.begin(), [](unsigned char c) { return (char)std::tolower(c); }); + auto authHeaderPatternIt = authHeaderAlternatives_.find(key); if (authHeaderPatternIt != authHeaderAlternatives_.end()) { if (std::regex_match(v, authHeaderPatternIt->second)) { return true; From 5126872ee950350c67999a4b77ccc40039b19bfb Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Fri, 23 Jan 2026 17:29:34 +0100 Subject: [PATCH 08/38] Use Drogon instead of uWebSockets+httplib. --- CMakeLists.txt | 14 +- cmake/deps.cmake | 142 ++-- libs/http-datasource/CMakeLists.txt | 3 +- .../include/mapget/detail/http-server.h | 10 +- .../http-datasource/datasource-client.h | 15 +- .../http-datasource/datasource-server.h | 2 +- .../http-datasource/src/datasource-client.cpp | 77 +- .../http-datasource/src/datasource-server.cpp | 204 ++--- libs/http-datasource/src/http-server.cpp | 243 ++---- libs/http-service/CMakeLists.txt | 3 +- .../mapget/http-service/http-service.h | 2 +- libs/http-service/src/http-client.cpp | 137 +-- libs/http-service/src/http-service.cpp | 780 ++++++++++-------- libs/pymapget/CMakeLists.txt | 12 +- test/unit/CMakeLists.txt | 15 +- test/unit/test-datasource-server.cpp | 70 ++ test/unit/test-http-datasource.cpp | 548 ++++++------ test/unit/test-http-service-fixture.h | 17 + test/unit/test-main.cpp | 50 ++ 19 files changed, 1305 insertions(+), 1039 deletions(-) create mode 100644 test/unit/test-datasource-server.cpp create mode 100644 test/unit/test-http-service-fixture.h create mode 100644 test/unit/test-main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1d73bb5b..f97065b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,10 @@ endif() set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) +if (WIN32) + add_compile_definitions(NOMINMAX) +endif() + include(FetchContent) include(GNUInstallDirs) @@ -22,12 +26,21 @@ if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) set(MAPGET_WITH_HTTPLIB ON CACHE BOOL "Enable mapget-http-datasource and mapget-http-service libraries.") set(MAPGET_ENABLE_TESTING ON CACHE BOOL "Enable testing.") set(MAPGET_BUILD_EXAMPLES ON CACHE BOOL "Build examples.") + # Prevent CPM dependencies (e.g. Drogon/Trantor) from registering their own + # tests into this project's ctest run. + set(BUILD_TESTING OFF CACHE BOOL "Disable dependency tests" FORCE) endif() option(MAPGET_WITH_WHEEL "Enable mapget Python wheel (output to WHEEL_DEPLOY_DIRECTORY).") option(MAPGET_WITH_SERVICE "Enable mapget-service library. Requires threads.") option(MAPGET_WITH_HTTPLIB "Enable mapget-http-datasource and mapget-http-service libraries.") +if (MAPGET_ENABLE_TESTING) + # Enable testing before adding CPM dependencies so stale/third-party CTest + # files don't linger in the build tree. + enable_testing() +endif() + set(Python3_FIND_STRATEGY LOCATION) if (NOT MSVC) @@ -112,7 +125,6 @@ endif() # tests if (MAPGET_ENABLE_TESTING) - enable_testing() add_subdirectory(test/unit) if (MAPGET_WITH_WHEEL) diff --git a/cmake/deps.cmake b/cmake/deps.cmake index a990b4a2..629fc978 100644 --- a/cmake/deps.cmake +++ b/cmake/deps.cmake @@ -25,6 +25,21 @@ CPMAddPackage( "BUILD_TESTING OFF") if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) + # OpenSSL's Configure script needs a "full" Perl distribution. Git for + # Windows ships a minimal perl that is missing required modules (e.g. + # Locale::Maketext::Simple), causing OpenSSL builds to fail. + if (WIN32) + if (NOT DEFINED PERL_EXECUTABLE OR PERL_EXECUTABLE MATCHES "[\\\\/]Git[\\\\/]usr[\\\\/]bin[\\\\/]perl\\.exe$") + find_program(_MAPGET_STRAWBERRY_PERL + NAMES perl.exe + PATHS "C:/Strawberry/perl/bin" + NO_DEFAULT_PATH) + if (_MAPGET_STRAWBERRY_PERL) + set(PERL_EXECUTABLE "${_MAPGET_STRAWBERRY_PERL}" CACHE FILEPATH "" FORCE) + endif() + endif() + endif() + set (OPENSSL_VERSION openssl-3.5.2) CPMAddPackage("gh:klebert-engineering/openssl-cmake@1.0.0") CPMAddPackage( @@ -40,19 +55,67 @@ if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) endif() CPMAddPackage( - URI "gh:yhirose/cpp-httplib@0.15.3" + NAME jsoncpp + GIT_REPOSITORY https://github.com/open-source-parsers/jsoncpp + GIT_TAG 1.9.5 + GIT_SHALLOW ON OPTIONS - "CPPHTTPLIB_USE_POLL ON" - "HTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN OFF" - "HTTPLIB_INSTALL OFF" - "HTTPLIB_USE_OPENSSL_IF_AVAILABLE OFF" - "HTTPLIB_USE_ZLIB_IF_AVAILABLE OFF") - # Manually enable openssl/zlib in httplib to avoid FindPackage calls. - target_compile_definitions(httplib INTERFACE - CPPHTTPLIB_OPENSSL_SUPPORT - CPPHTTPLIB_ZLIB_SUPPORT) - target_link_libraries(httplib INTERFACE - OpenSSL::SSL OpenSSL::Crypto ZLIB::ZLIB) + "JSONCPP_WITH_TESTS OFF" + "JSONCPP_WITH_POST_BUILD_UNITTEST OFF" + "JSONCPP_WITH_PKGCONFIG_SUPPORT OFF" + "JSONCPP_WITH_CMAKE_PACKAGE OFF" + "BUILD_SHARED_LIBS OFF" + "BUILD_STATIC_LIBS ON" + "BUILD_OBJECT_LIBS OFF") + # Help Drogon's FindJsoncpp.cmake locate jsoncpp when built via CPM. + set(JSONCPP_INCLUDE_DIRS "${jsoncpp_SOURCE_DIR}/include" CACHE PATH "" FORCE) + set(JSONCPP_LIBRARIES jsoncpp_static CACHE STRING "" FORCE) + # CPM generates a dummy package redirect config at + # `${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}/jsoncpp-config.cmake`. Drogon uses + # `find_package(Jsoncpp)` (config-first), so make that redirect actually + # define the expected `Jsoncpp_lib` target. + if (DEFINED CMAKE_FIND_PACKAGE_REDIRECTS_DIR) + file(MAKE_DIRECTORY "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}") + file(WRITE "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}/jsoncpp-extra.cmake" [=[ +if(NOT TARGET Jsoncpp_lib) + add_library(Jsoncpp_lib INTERFACE) + target_include_directories(Jsoncpp_lib INTERFACE "${JSONCPP_INCLUDE_DIRS}") + target_link_libraries(Jsoncpp_lib INTERFACE ${JSONCPP_LIBRARIES}) +endif() +]=]) + endif() + + # Drogon defines install(EXPORT ...) rules unconditionally, which fail when + # used as a subproject with CPM-provided dependencies (zlib/jsoncpp/etc). + # Since mapget only needs Drogon for building, temporarily suppress install + # rule generation while configuring Drogon. + set(_MAPGET_PREV_SKIP_INSTALL_RULES "${CMAKE_SKIP_INSTALL_RULES}") + if (DEFINED BUILD_TESTING) + set(_MAPGET_PREV_BUILD_TESTING "${BUILD_TESTING}") + endif() + set(CMAKE_SKIP_INSTALL_RULES ON) + set(BUILD_TESTING OFF) + + CPMAddPackage( + URI "gh:drogonframework/drogon@1.9.7" + OPTIONS + "BUILD_CTL OFF" + "BUILD_EXAMPLES OFF" + "BUILD_ORM OFF" + "BUILD_BROTLI OFF" + "BUILD_YAML_CONFIG OFF" + "BUILD_SHARED_LIBS OFF" + "USE_SUBMODULE ON" + "USE_STATIC_LIBS_ONLY OFF" + "USE_POSTGRESQL OFF" + "USE_MYSQL OFF" + "USE_SQLITE3 OFF" + GIT_SUBMODULES "trantor") + + set(CMAKE_SKIP_INSTALL_RULES "${_MAPGET_PREV_SKIP_INSTALL_RULES}") + if (DEFINED _MAPGET_PREV_BUILD_TESTING) + set(BUILD_TESTING "${_MAPGET_PREV_BUILD_TESTING}") + endif() CPMAddPackage( URI "gh:jbeder/yaml-cpp#aa8d4e@0.8.0" # Use > 0.8.0 once available. @@ -65,59 +128,6 @@ if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) CPMAddPackage("gh:pboettch/json-schema-validator#2.3.0") CPMAddPackage("gh:okdshin/PicoSHA2@1.0.1") - if (WIN32) - CPMAddPackage( - NAME libuv - GIT_REPOSITORY https://github.com/libuv/libuv - GIT_TAG v1.48.0 - GIT_SHALLOW ON - OPTIONS - "LIBUV_BUILD_TESTS OFF" - "LIBUV_BUILD_BENCH OFF" - "LIBUV_BUILD_SHARED OFF" - "LIBUV_BUILD_EXAMPLES OFF") - endif() - - CPMAddPackage( - NAME uSockets - GIT_REPOSITORY https://github.com/uNetworking/uSockets - GIT_TAG v0.8.5 - GIT_SHALLOW ON - GIT_SUBMODULES "") - if (NOT TARGET uSockets) - file(GLOB_RECURSE U_SOCKETS_SOURCES CONFIGURE_DEPENDS - "${uSockets_SOURCE_DIR}/src/*.c" - "${uSockets_SOURCE_DIR}/src/*.cpp") - add_library(uSockets STATIC ${U_SOCKETS_SOURCES}) - target_include_directories(uSockets PUBLIC "${uSockets_SOURCE_DIR}/src") - target_compile_definitions(uSockets PRIVATE LIBUS_USE_OPENSSL) - target_link_libraries(uSockets PUBLIC OpenSSL::SSL OpenSSL::Crypto) - if (WIN32) - target_link_libraries(uSockets PUBLIC ws2_32) - if (TARGET uv_a) - target_link_libraries(uSockets PUBLIC uv_a) - elseif (TARGET uv) - target_link_libraries(uSockets PUBLIC uv) - else() - message(FATAL_ERROR "libuv was requested for uSockets on Windows, but no CMake target (uv_a/uv) was found.") - endif() - endif() - endif() - - CPMAddPackage( - NAME uWebSockets - GIT_REPOSITORY https://github.com/uNetworking/uWebSockets - GIT_TAG v20.37.0 - GIT_SHALLOW ON - GIT_SUBMODULES "") - if (NOT TARGET uWebSockets) - add_library(uWebSockets INTERFACE) - target_include_directories(uWebSockets INTERFACE "${uWebSockets_SOURCE_DIR}/src") - target_link_libraries(uWebSockets INTERFACE uSockets ZLIB::ZLIB) - if (CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") - target_compile_options(uWebSockets INTERFACE -Wno-deprecated-declarations) - endif() - endif() endif () if (MAPGET_WITH_WHEEL AND NOT TARGET pybind11) @@ -130,7 +140,7 @@ if (MAPGET_WITH_SERVICE OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) endif() if (MAPGET_WITH_WHEEL AND NOT TARGET python-cmake-wheel) - CPMAddPackage("gh:Klebert-Engineering/python-cmake-wheel@1.1.0") + CPMAddPackage("gh:Klebert-Engineering/python-cmake-wheel@1.2.0") endif() if (MAPGET_ENABLE_TESTING) diff --git a/libs/http-datasource/CMakeLists.txt b/libs/http-datasource/CMakeLists.txt index 917eded2..7b323329 100644 --- a/libs/http-datasource/CMakeLists.txt +++ b/libs/http-datasource/CMakeLists.txt @@ -17,8 +17,7 @@ target_include_directories(mapget-http-datasource target_link_libraries(mapget-http-datasource PUBLIC - httplib::httplib - uWebSockets + drogon mapget-model mapget-service tiny-process-library) diff --git a/libs/http-datasource/include/mapget/detail/http-server.h b/libs/http-datasource/include/mapget/detail/http-server.h index bcc9656f..cc6db8cb 100644 --- a/libs/http-datasource/include/mapget/detail/http-server.h +++ b/libs/http-datasource/include/mapget/detail/http-server.h @@ -1,12 +1,12 @@ #pragma once +#include #include #include -// Forward declare uWebSockets app type to avoid including uWS headers in public headers. -namespace uWS { -template struct TemplatedApp; -using App = TemplatedApp; +// Forward declare Drogon app type to avoid including drogon headers in public headers. +namespace drogon { +class HttpAppFramework; } namespace mapget { @@ -77,7 +77,7 @@ class HttpServer * This function is called upon the first call to go(), * and allows any derived server class to add endpoints. */ - virtual void setup(uWS::App&) = 0; + virtual void setup(drogon::HttpAppFramework&) = 0; /** * Derived servers can use this to control whether diff --git a/libs/http-datasource/include/mapget/http-datasource/datasource-client.h b/libs/http-datasource/include/mapget/http-datasource/datasource-client.h index 5bb03aee..91038d73 100644 --- a/libs/http-datasource/include/mapget/http-datasource/datasource-client.h +++ b/libs/http-datasource/include/mapget/http-datasource/datasource-client.h @@ -3,15 +3,24 @@ #include "mapget/model/sourcedatalayer.h" #include "mapget/model/featurelayer.h" #include "mapget/service/datasource.h" -#include "httplib.h" #include +#include #include +#include namespace TinyProcessLib { class Process; } +namespace drogon { +class HttpClient; +} + +namespace trantor { +class EventLoopThread; +} + namespace mapget { @@ -32,6 +41,7 @@ class RemoteDataSource : public DataSource * fails for any reason. */ RemoteDataSource(std::string const& host, uint16_t port); + ~RemoteDataSource(); // DataSource method overrides DataSourceInfo info() override; @@ -48,7 +58,8 @@ class RemoteDataSource : public DataSource std::string error_; // Multiple http clients allow parallel GET requests - std::vector httpClients_; + std::unique_ptr httpClientLoop_; + std::vector> httpClients_; std::atomic_uint64_t nextClient_{0}; }; diff --git a/libs/http-datasource/include/mapget/http-datasource/datasource-server.h b/libs/http-datasource/include/mapget/http-datasource/datasource-server.h index 501f01ac..a0deda49 100644 --- a/libs/http-datasource/include/mapget/http-datasource/datasource-server.h +++ b/libs/http-datasource/include/mapget/http-datasource/datasource-server.h @@ -50,7 +50,7 @@ class DataSourceServer : public HttpServer DataSourceInfo const& info(); private: - void setup(uWS::App&) override; + void setup(drogon::HttpAppFramework&) override; struct Impl; std::unique_ptr impl_; diff --git a/libs/http-datasource/src/datasource-client.cpp b/libs/http-datasource/src/datasource-client.cpp index 1199c48c..128a5446 100644 --- a/libs/http-datasource/src/datasource-client.cpp +++ b/libs/http-datasource/src/datasource-client.cpp @@ -3,6 +3,10 @@ #include "process.hpp" #include "mapget/log.h" +#include +#include +#include + #include #include @@ -11,25 +15,43 @@ namespace mapget RemoteDataSource::RemoteDataSource(const std::string& host, uint16_t port) { + httpClientLoop_ = std::make_unique("MapgetRemoteDataSource"); + httpClientLoop_->run(); + + const auto hostString = fmt::format("http://{}:{}/", host, port); + // Fetch data source info. - httplib::Client client(host, port); - auto fetchedInfoJson = client.Get("/info"); - if (!fetchedInfoJson || fetchedInfoJson->status >= 300) - raise("Failed to fetch datasource info."); - info_ = DataSourceInfo::fromJson(nlohmann::json::parse(fetchedInfoJson->body)); + auto infoClient = drogon::HttpClient::newHttpClient(hostString, httpClientLoop_->getLoop()); + auto infoReq = drogon::HttpRequest::newHttpRequest(); + infoReq->setMethod(drogon::Get); + infoReq->setPath("/info"); + + auto [result, fetchedInfoResp] = infoClient->sendRequest(infoReq); + if (result != drogon::ReqResult::Ok || !fetchedInfoResp) { + raise(fmt::format("Failed to fetch datasource info: [{}]", drogon::to_string_view(result))); + } + if ((int)fetchedInfoResp->statusCode() >= 300) { + raise(fmt::format("Failed to fetch datasource info: [{}]", (int)fetchedInfoResp->statusCode())); + } + info_ = DataSourceInfo::fromJson(nlohmann::json::parse(std::string(fetchedInfoResp->body()))); if (info_.nodeId_.empty()) { // Unique node IDs are required for the string pool offsets. raise( fmt::format("Remote data source is missing node ID! Source info: {}", - fetchedInfoJson->body)); + std::string(fetchedInfoResp->body()))); } // Create as many clients as parallel requests are allowed. - for (auto i = 0; i < std::max(info_.maxParallelJobs_, 1); ++i) - httpClients_.emplace_back(host, port); + const auto clientCount = (std::max)(info_.maxParallelJobs_, 1); + httpClients_.reserve(clientCount); + for (auto i = 0; i < clientCount; ++i) { + httpClients_.emplace_back(drogon::HttpClient::newHttpClient(hostString, httpClientLoop_->getLoop())); + } } +RemoteDataSource::~RemoteDataSource() = default; + DataSourceInfo RemoteDataSource::info() { return info_; @@ -54,30 +76,26 @@ RemoteDataSource::get(const MapTileKey& k, Cache::Ptr& cache, const DataSourceIn auto& client = httpClients_[(nextClient_++) % httpClients_.size()]; // Send a GET tile request. - auto tileResponse = client.Get(fmt::format( + auto tileReq = drogon::HttpRequest::newHttpRequest(); + tileReq->setMethod(drogon::Get); + tileReq->setPath(fmt::format( "/tile?layer={}&tileId={}&stringPoolOffset={}", k.layerId_, k.tileId_.value_, cachedStringPoolOffset(info.nodeId_, cache))); + auto [resultCode, tileResponse] = client->sendRequest(tileReq); // Check that the response is OK. - if (!tileResponse || tileResponse->status >= 300) { + if (resultCode != drogon::ReqResult::Ok || !tileResponse || (int)tileResponse->statusCode() >= 300) { // Forward to base class get(). This will instantiate a // default TileLayer and call fill(). In our implementation // of fill, we set an error. - if (tileResponse) { - if (tileResponse->has_header("HTTPLIB_ERROR")) { - error_ = tileResponse->get_header_value("HTTPLIB_ERROR"); - } - else if (tileResponse->has_header("EXCEPTION_WHAT")) { - error_ = tileResponse->get_header_value("EXCEPTION_WHAT"); - } - else { - error_ = fmt::format("Code {}", tileResponse->status); - } - } - else { + if (resultCode != drogon::ReqResult::Ok) { + error_ = drogon::to_string(resultCode); + } else if (tileResponse) { + error_ = fmt::format("Code {}", (int)tileResponse->statusCode()); + } else { error_ = "No remote response."; } @@ -92,7 +110,7 @@ RemoteDataSource::get(const MapTileKey& k, Cache::Ptr& cache, const DataSourceIn [&](auto&& mapId, auto&& layerId) { return info.getLayer(std::string(layerId)); }, [&](auto&& tile) { result = tile; }, cache); - reader.read(tileResponse->body); + reader.read(std::string(tileResponse->body())); return result; } @@ -102,12 +120,15 @@ std::vector RemoteDataSource::locate(const LocateRequest& req) // Round-robin usage of http clients to facilitate parallel requests. auto& client = httpClients_[(nextClient_++) % httpClients_.size()]; - // Send a GET tile request. - auto locateResponse = client.Post( - fmt::format("/locate"), req.serialize().dump(), "application/json"); + auto locateReq = drogon::HttpRequest::newHttpRequest(); + locateReq->setMethod(drogon::Post); + locateReq->setPath("/locate"); + locateReq->setContentTypeCode(drogon::CT_APPLICATION_JSON); + locateReq->setBody(req.serialize().dump()); + auto [resultCode, locateResponse] = client->sendRequest(locateReq); // Check that the response is OK. - if (!locateResponse || locateResponse->status >= 300) { + if (resultCode != drogon::ReqResult::Ok || !locateResponse || (int)locateResponse->statusCode() >= 300) { // Forward to base class get(). This will instantiate a // default TileFeatureLayer and call fill(). In our implementation // of fill, we set an error. @@ -116,7 +137,7 @@ std::vector RemoteDataSource::locate(const LocateRequest& req) } // Check the response body for expected content. - auto responseJson = nlohmann::json::parse(locateResponse->body); + auto responseJson = nlohmann::json::parse(std::string(locateResponse->body())); if (responseJson.is_null()) { return {}; } diff --git a/libs/http-datasource/src/datasource-server.cpp b/libs/http-datasource/src/datasource-server.cpp index a8435058..30f83794 100644 --- a/libs/http-datasource/src/datasource-server.cpp +++ b/libs/http-datasource/src/datasource-server.cpp @@ -4,7 +4,8 @@ #include "mapget/model/info.h" #include "mapget/model/stream.h" -#include +#include +#include #include #include @@ -60,98 +61,107 @@ DataSourceServer& DataSourceServer::onLocateRequest( DataSourceInfo const& DataSourceServer::info() { return impl_->info_; } -void DataSourceServer::setup(uWS::App& app) +void DataSourceServer::setup(drogon::HttpAppFramework& app) { - app.get("/tile", [this](auto* res, auto* req) { - try { - auto layerIdParam = req->getQuery("layer"); - auto tileIdParam = req->getQuery("tileId"); - - if (layerIdParam.empty() || tileIdParam.empty()) { - res->writeStatus("400 Bad Request"); - res->writeHeader("Content-Type", "text/plain"); - res->end("Missing query parameter: layer and/or tileId"); - return; - } + app.registerHandler( + "/tile", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) + { + try { + auto const& layerIdParam = req->getParameter("layer"); + auto const& tileIdParam = req->getParameter("tileId"); + + if (layerIdParam.empty() || tileIdParam.empty()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Missing query parameter: layer and/or tileId"); + callback(resp); + return; + } - auto layer = impl_->info_.getLayer(std::string(layerIdParam)); + auto layer = impl_->info_.getLayer(layerIdParam); + TileId tileId{std::stoull(tileIdParam)}; - TileId tileId{std::stoull(std::string(tileIdParam))}; + auto stringPoolOffsetParam = (simfil::StringId)0; + auto const& stringPoolOffsetStr = req->getParameter("stringPoolOffset"); + if (!stringPoolOffsetStr.empty()) { + stringPoolOffsetParam = (simfil::StringId)std::stoul(stringPoolOffsetStr); + } - auto stringPoolOffsetParam = (simfil::StringId)0; - auto stringPoolOffsetStr = req->getQuery("stringPoolOffset"); - if (!stringPoolOffsetStr.empty()) { - stringPoolOffsetParam = (simfil::StringId)std::stoul(std::string(stringPoolOffsetStr)); - } + std::string responseType = "binary"; + auto const& responseTypeStr = req->getParameter("responseType"); + if (!responseTypeStr.empty()) + responseType = responseTypeStr; + + auto tileLayer = [&]() -> std::shared_ptr + { + switch (layer->type_) { + case mapget::LayerType::Features: { + auto tileFeatureLayer = std::make_shared( + tileId, impl_->info_.nodeId_, impl_->info_.mapId_, layer, impl_->strings_); + impl_->tileFeatureCallback_(tileFeatureLayer); + return tileFeatureLayer; + } + case mapget::LayerType::SourceData: { + auto tileSourceLayer = std::make_shared( + tileId, impl_->info_.nodeId_, impl_->info_.mapId_, layer, impl_->strings_); + impl_->tileSourceDataCallback_(tileSourceLayer); + return tileSourceLayer; + } + default: + throw std::runtime_error(fmt::format("Unsupported layer type {}", (int)layer->type_)); + } + }(); + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + + if (responseType == "binary") { + std::string content; + TileLayerStream::StringPoolOffsetMap stringPoolOffsets{{impl_->info_.nodeId_, stringPoolOffsetParam}}; + TileLayerStream::Writer layerWriter{ + [&](std::string bytes, TileLayerStream::MessageType) { content.append(bytes); }, + stringPoolOffsets}; + layerWriter.write(tileLayer); + + resp->setContentTypeString("application/binary"); + resp->setBody(std::move(content)); + } else { + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(tileLayer->toJson().dump()); + } - std::string responseType = "binary"; - auto responseTypeStr = req->getQuery("responseType"); - if (!responseTypeStr.empty()) { - responseType = std::string(responseTypeStr); + callback(resp); } - - auto tileLayer = [&]() -> std::shared_ptr { - switch (layer->type_) { - case mapget::LayerType::Features: { - auto tileFeatureLayer = std::make_shared( - tileId, impl_->info_.nodeId_, impl_->info_.mapId_, layer, impl_->strings_); - impl_->tileFeatureCallback_(tileFeatureLayer); - return tileFeatureLayer; - } - case mapget::LayerType::SourceData: { - auto tileSourceLayer = std::make_shared( - tileId, impl_->info_.nodeId_, impl_->info_.mapId_, layer, impl_->strings_); - impl_->tileSourceDataCallback_(tileSourceLayer); - return tileSourceLayer; - } - default: - throw std::runtime_error(fmt::format("Unsupported layer type {}", (int)layer->type_)); - } - }(); - - if (responseType == "binary") { - std::string content; - TileLayerStream::StringPoolOffsetMap stringPoolOffsets{ - {impl_->info_.nodeId_, stringPoolOffsetParam}}; - TileLayerStream::Writer layerWriter{ - [&](std::string bytes, TileLayerStream::MessageType) { content.append(bytes); }, - stringPoolOffsets}; - layerWriter.write(tileLayer); - - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "application/binary"); - res->end(content); - } else { - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "application/json"); - res->end(tileLayer->toJson().dump()); + catch (std::exception const& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Error: ") + e.what()); + callback(resp); } - } - catch (std::exception const& e) { - res->writeStatus("500 Internal Server Error"); - res->writeHeader("Content-Type", "text/plain"); - res->end(std::string("Error: ") + e.what()); - } - }); - - app.get("/info", [this](auto* res, auto* /*req*/) { - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "application/json"); - res->end(impl_->info_.toJson().dump()); - }); - - app.post("/locate", [this](auto* res, auto* /*req*/) { - auto aborted = std::make_shared(false); - res->onAborted([aborted]() { *aborted = true; }); - - res->onData([this, res, aborted, body = std::string()](std::string_view chunk, bool last) mutable { - if (*aborted) - return; - body.append(chunk.data(), chunk.size()); - if (!last) - return; + }, + {drogon::Get}); + + app.registerHandler( + "/info", + [this](const drogon::HttpRequestPtr&, std::function&& callback) + { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(impl_->info_.toJson().dump()); + callback(resp); + }, + {drogon::Get}); + + app.registerHandler( + "/locate", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) + { try { - LocateRequest parsedReq(nlohmann::json::parse(body)); + LocateRequest parsedReq(nlohmann::json::parse(std::string(req->body()))); auto responseJson = nlohmann::json::array(); if (impl_->locateCallback_) { @@ -160,21 +170,21 @@ void DataSourceServer::setup(uWS::App& app) } } - if (*aborted) - return; - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "application/json"); - res->end(responseJson.dump()); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(responseJson.dump()); + callback(resp); } catch (std::exception const& e) { - if (*aborted) - return; - res->writeStatus("400 Bad Request"); - res->writeHeader("Content-Type", "text/plain"); - res->end(std::string("Invalid request: ") + e.what()); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid request: ") + e.what()); + callback(resp); } - }); - }); + }, + {drogon::Post}); } } // namespace mapget diff --git a/libs/http-datasource/src/http-server.cpp b/libs/http-datasource/src/http-server.cpp index cdceab2c..f7106509 100644 --- a/libs/http-datasource/src/http-server.cpp +++ b/libs/http-datasource/src/http-server.cpp @@ -1,19 +1,17 @@ #include "mapget/detail/http-server.h" #include "mapget/log.h" -#include -#include +#include +#include #include #include +#include #include #include #include -#include #include #include -#include -#include #include #include #include @@ -24,20 +22,28 @@ namespace mapget { -// initialize the atomic activeHttpServer with nullptr +// Used by waitForSignal() so the signal handler knows what to stop. static std::atomic activeHttpServer = nullptr; +// Drogon uses a singleton app instance; running multiple independent servers +// in-process is not supported. +static std::atomic activeDrogonServer = nullptr; + namespace { + struct MountPoint { std::string urlPrefix; std::filesystem::path fsRoot; }; -[[nodiscard]] bool startsWith(std::string_view s, std::string_view prefix) +[[nodiscard]] bool looksLikeWindowsDrivePath(std::string_view s) { - return s.size() >= prefix.size() && s.substr(0, prefix.size()) == prefix; + if (s.size() < 3) + return false; + const unsigned char drive = static_cast(s[0]); + return std::isalpha(drive) && s[1] == ':' && (s[2] == '\\' || s[2] == '/'); } [[nodiscard]] std::string normalizeUrlPrefix(std::string prefix) @@ -51,75 +57,6 @@ struct MountPoint return prefix; } -[[nodiscard]] std::string_view guessMimeType(std::filesystem::path const& filePath) -{ - auto ext = filePath.extension().string(); - std::ranges::transform(ext, ext.begin(), [](unsigned char c) { return (char)std::tolower(c); }); - - if (ext == ".html" || ext == ".htm") - return "text/html"; - if (ext == ".css") - return "text/css"; - if (ext == ".js") - return "application/javascript"; - if (ext == ".json") - return "application/json"; - if (ext == ".svg") - return "image/svg+xml"; - if (ext == ".png") - return "image/png"; - if (ext == ".jpg" || ext == ".jpeg") - return "image/jpeg"; - if (ext == ".ico") - return "image/x-icon"; - if (ext == ".woff2") - return "font/woff2"; - if (ext == ".woff") - return "font/woff"; - if (ext == ".ttf") - return "font/ttf"; - if (ext == ".txt") - return "text/plain"; - - return "application/octet-stream"; -} - -[[nodiscard]] std::optional resolveStaticFile( - std::vector const& mounts, - std::string_view urlPath) -{ - if (mounts.empty()) - return std::nullopt; - if (!startsWith(urlPath, "/")) - return std::nullopt; - - // Longest-prefix match. - MountPoint const* best = nullptr; - for (auto const& m : mounts) { - if (startsWith(urlPath, m.urlPrefix) && (!best || m.urlPrefix.size() > best->urlPrefix.size())) - best = &m; - } - if (!best) - return std::nullopt; - - std::string_view remainder = urlPath.substr(best->urlPrefix.size()); - if (!remainder.empty() && remainder.front() == '/') - remainder.remove_prefix(1); - - std::filesystem::path relativePath = std::filesystem::path(std::string(remainder)).lexically_normal(); - if (relativePath.empty() || urlPath.back() == '/') - relativePath /= "index.html"; - - // Basic path traversal protection: reject any ".." segments. - for (auto const& part : relativePath) { - if (part == "..") - return std::nullopt; - } - - std::filesystem::path candidate = (best->fsRoot / relativePath).lexically_normal(); - return candidate; -} - } // namespace struct HttpServer::Impl @@ -134,20 +71,14 @@ struct HttpServer::Impl uint16_t port_ = 0; bool printPortToStdout_ = false; + bool startedOnce_ = false; std::mutex mountsMutex_; std::vector mounts_; - uWS::Loop* loop_ = nullptr; - us_listen_socket_t* listenSocket_ = nullptr; - static void handleSignal(int) { - // Temporarily holds the current active HttpServer auto* expected = activeHttpServer.load(); - - // Stop the active instance when a signal is received. - // We use compare_exchange_strong to make the operation atomic. if (activeHttpServer.compare_exchange_strong(expected, nullptr)) { if (expected) { expected->stop(); @@ -168,14 +99,21 @@ HttpServer::HttpServer() : impl_(new Impl()) {} HttpServer::~HttpServer() { - if (isRunning()) - stop(); + stop(); } void HttpServer::go(std::string const& interfaceAddr, uint16_t port, uint32_t waitMs) { if (impl_->running_ || impl_->serverThread_.joinable()) raise("HttpServer is already running"); + if (impl_->startedOnce_) + raise("HttpServer cannot be restarted in-process (Drogon singleton)"); + + HttpServer* expected = nullptr; + if (!activeDrogonServer.compare_exchange_strong(expected, this)) + raise("Only one HttpServer can run per process (Drogon singleton)"); + + impl_->startedOnce_ = true; // Reset start state. { @@ -188,12 +126,9 @@ void HttpServer::go(std::string const& interfaceAddr, uint16_t port, uint32_t wa [this, interfaceAddr, port] { try { - uWS::App app; + auto& app = drogon::app(); - // Allow derived class to set up the server - setup(app); - - // Copy mounts to avoid locking in the hot path. + // Copy mounts to avoid locking after the server thread starts. std::vector mountsCopy; { std::lock_guard lock(impl_->mountsMutex_); @@ -201,60 +136,36 @@ void HttpServer::go(std::string const& interfaceAddr, uint16_t port, uint32_t wa } if (!mountsCopy.empty()) { - app.get( - "/*", - [mounts = std::move(mountsCopy)](auto* res, auto* req) mutable - { - auto urlPath = req->getUrl(); - auto candidate = resolveStaticFile(mounts, urlPath); - if (!candidate || !std::filesystem::exists(*candidate) || - !std::filesystem::is_regular_file(*candidate)) { - res->writeStatus("404 Not Found"); - res->writeHeader("Content-Type", "text/plain"); - res->end("Not found"); - return; - } - - std::ifstream ifs(*candidate, std::ios::binary); - if (!ifs) { - res->writeStatus("500 Internal Server Error"); - res->writeHeader("Content-Type", "text/plain"); - res->end("Failed to open file"); - return; - } - - std::string content; - ifs.seekg(0, std::ios::end); - content.resize(static_cast(ifs.tellg())); - ifs.seekg(0, std::ios::beg); - if (!content.empty()) { - ifs.read(content.data(), static_cast(content.size())); - } - - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", guessMimeType(*candidate)); - res->end(content); - }); + std::sort( + mountsCopy.begin(), + mountsCopy.end(), + [](MountPoint const& a, MountPoint const& b) { return a.urlPrefix.size() > b.urlPrefix.size(); }); + + // Using empty document root makes addALocation's "alias" parameter + // work with absolute Windows paths (e.g. "C:/path"). + app.setDocumentRoot(""); + + for (auto const& m : mountsCopy) { + app.addALocation(m.urlPrefix, "", m.fsRoot.generic_string()); + } } - app.listen( - interfaceAddr, - port, - [this, interfaceAddr, port](us_listen_socket_t* listenSocket) - { - if (!listenSocket) { - impl_->notifyStart( - fmt::format("Could not start HttpServer on {}:{}", interfaceAddr, port)); - return; - } + // Allow derived class to set up the server. + setup(app); - impl_->listenSocket_ = listenSocket; - impl_->loop_ = uWS::Loop::get(); + app.addListener(interfaceAddr, port); - // Determine actual port (port may be 0 for ephemeral). - impl_->port_ = static_cast( - us_socket_local_port(0, reinterpret_cast(listenSocket))); + app.registerBeginningAdvice([this]() { + // Beginning advice runs before listeners start. Post the actual + // startup notification to run after startListening() completed. + drogon::app().getLoop()->queueInLoop([this]() { + auto listeners = drogon::app().getListeners(); + if (listeners.empty()) { + impl_->notifyStart("HttpServer started without listeners"); + return; + } + impl_->port_ = listeners.front().toPort(); impl_->running_ = true; impl_->notifyStart(); @@ -263,14 +174,7 @@ void HttpServer::go(std::string const& interfaceAddr, uint16_t port, uint32_t wa else log().info("====== Running on port {} ======", impl_->port_); }); - - // If listen failed, exit without running the loop. - if (!impl_->running_) { - if (!impl_->startNotified_) { - impl_->notifyStart(fmt::format("Could not start HttpServer on {}:{}", interfaceAddr, port)); - } - return; - } + }); app.run(); } @@ -279,8 +183,9 @@ void HttpServer::go(std::string const& interfaceAddr, uint16_t port, uint32_t wa } impl_->running_ = false; - impl_->listenSocket_ = nullptr; - impl_->loop_ = nullptr; + + HttpServer* expected = this; + (void)activeDrogonServer.compare_exchange_strong(expected, nullptr); }); std::unique_lock lk(impl_->startMutex_); @@ -305,10 +210,8 @@ void HttpServer::stop() if (!impl_->serverThread_.joinable()) return; - if (impl_->loop_ && impl_->listenSocket_) { - auto* loop = impl_->loop_; - auto* listenSocket = impl_->listenSocket_; - loop->defer([listenSocket]() { us_listen_socket_close(0, listenSocket); }); + if (drogon::app().isRunning()) { + drogon::app().quit(); } if (impl_->serverThread_.get_id() != std::this_thread::get_id()) @@ -322,14 +225,11 @@ uint16_t HttpServer::port() const void HttpServer::waitForSignal() { - // So the signal handler knows what to call activeHttpServer = this; - // Set the signal handler for SIGINT and SIGTERM. std::signal(SIGINT, Impl::handleSignal); std::signal(SIGTERM, Impl::handleSignal); - // Wait for the signal handler to stop us, or the server to shut down on its own. while (isRunning()) { std::this_thread::sleep_for(std::chrono::milliseconds(200)); } @@ -339,26 +239,29 @@ void HttpServer::waitForSignal() bool HttpServer::mountFileSystem(std::string const& pathFromTo) { - using namespace std::ranges; - auto parts = pathFromTo | views::split(':') | - views::transform([](auto&& s) { return std::string(&*s.begin(), distance(s)); }); - auto partsVec = std::vector(parts.begin(), parts.end()); - std::string urlPrefix; - std::filesystem::path fsRoot; - if (partsVec.size() == 1) { + std::string fsRootStr; + + const auto firstColon = pathFromTo.find(':'); + if (firstColon == std::string::npos || looksLikeWindowsDrivePath(pathFromTo)) { urlPrefix = "/"; - fsRoot = partsVec[0]; - } else if (partsVec.size() == 2) { - urlPrefix = partsVec[0]; - fsRoot = partsVec[1]; + fsRootStr = pathFromTo; } else { - return false; + urlPrefix = pathFromTo.substr(0, firstColon); + fsRootStr = pathFromTo.substr(firstColon + 1); + if (fsRootStr.empty()) + return false; } urlPrefix = normalizeUrlPrefix(std::move(urlPrefix)); - if (!std::filesystem::exists(fsRoot) || !std::filesystem::is_directory(fsRoot)) + std::filesystem::path fsRoot(fsRootStr); + std::error_code ec; + fsRoot = std::filesystem::absolute(fsRoot, ec); + if (ec) + return false; + + if (!std::filesystem::exists(fsRoot, ec) || ec || !std::filesystem::is_directory(fsRoot, ec) || ec) return false; std::lock_guard lock(impl_->mountsMutex_); diff --git a/libs/http-service/CMakeLists.txt b/libs/http-service/CMakeLists.txt index 23c2f847..b9e8799f 100644 --- a/libs/http-service/CMakeLists.txt +++ b/libs/http-service/CMakeLists.txt @@ -17,8 +17,7 @@ target_include_directories(mapget-http-service target_link_libraries(mapget-http-service PUBLIC - httplib::httplib - uWebSockets + drogon yaml-cpp CLI11::CLI11 nlohmann_json_schema_validator diff --git a/libs/http-service/include/mapget/http-service/http-service.h b/libs/http-service/include/mapget/http-service/http-service.h index 1e87225d..61ea4c8e 100644 --- a/libs/http-service/include/mapget/http-service/http-service.h +++ b/libs/http-service/include/mapget/http-service/http-service.h @@ -55,7 +55,7 @@ class HttpService : public HttpServer, public Service ~HttpService() override; protected: - void setup(uWS::App& app) override; + void setup(drogon::HttpAppFramework& app) override; private: struct Impl; diff --git a/libs/http-service/src/http-client.cpp b/libs/http-service/src/http-client.cpp index 95ebdbea..ea6bb04e 100644 --- a/libs/http-service/src/http-client.cpp +++ b/libs/http-service/src/http-client.cpp @@ -1,51 +1,72 @@ #include "http-client.h" -#include "httplib.h" + #include "mapget/log.h" +#include +#include +#include + +#include + +#include "fmt/format.h" + namespace mapget { +namespace +{ + +void applyHeaders(drogon::HttpRequestPtr const& req, AuthHeaders const& headers) +{ + for (auto const& [k, v] : headers) { + req->addHeader(k, v); + } +} + +} // namespace + struct HttpClient::Impl { - httplib::Client client_; + std::unique_ptr loopThread_; + drogon::HttpClientPtr client_; std::unordered_map sources_; std::shared_ptr stringPoolProvider_; - httplib::Headers headers_; + AuthHeaders headers_; - Impl(std::string const& host, uint16_t port, AuthHeaders headers, bool enableCompression) : - client_(host, port), - headers_() + Impl(std::string const& host, uint16_t port, AuthHeaders headers, bool enableCompression) : headers_(std::move(headers)) { - for (auto const& [k, v] : headers) { - headers_.emplace(k, v); - } - // Add Accept-Encoding header if compression is enabled and not already present - if (enableCompression) { - bool hasAcceptEncoding = false; - for (const auto& [key, value] : headers_) { - if (key == "Accept-Encoding") { - hasAcceptEncoding = true; - break; - } - } - if (!hasAcceptEncoding) { - headers_.emplace("Accept-Encoding", "gzip"); - } + if (enableCompression && !(headers_.contains("Accept-Encoding") || headers_.contains("accept-encoding"))) { + headers_.emplace("Accept-Encoding", "gzip"); } - + + loopThread_ = std::make_unique("MapgetHttpClient"); + loopThread_->run(); + + const auto hostString = fmt::format("http://{}:{}/", host, port); + client_ = drogon::HttpClient::newHttpClient(hostString, loopThread_->getLoop()); + stringPoolProvider_ = std::make_shared(); - client_.set_keep_alive(false); - auto sourcesJson = client_.Get("/sources", headers_); - if (!sourcesJson || sourcesJson->status != 200) - raise( - fmt::format("Failed to fetch sources: [{}]", sourcesJson->status)); - for (auto const& info : nlohmann::json::parse(sourcesJson->body)) { + + // Fetch data sources (/sources). + auto req = drogon::HttpRequest::newHttpRequest(); + req->setMethod(drogon::Get); + req->setPath("/sources"); + applyHeaders(req, headers_); + + auto [result, resp] = client_->sendRequest(req); + if (result != drogon::ReqResult::Ok || !resp) { + raise(fmt::format("Failed to fetch sources: [{}]", drogon::to_string_view(result))); + } + if (resp->statusCode() != drogon::k200OK) { + raise(fmt::format("Failed to fetch sources: [{}]", (int)resp->statusCode())); + } + + for (auto const& info : nlohmann::json::parse(std::string(resp->body()))) { auto parsedInfo = DataSourceInfo::fromJson(info); sources_.emplace(parsedInfo.mapId_, parsedInfo); } } - [[nodiscard]] std::shared_ptr - resolve(std::string_view const& map, std::string_view const& layer) const + [[nodiscard]] std::shared_ptr resolve(std::string_view const& map, std::string_view const& layer) const { auto mapIt = sources_.find(std::string(map)); if (mapIt == sources_.end()) @@ -54,8 +75,10 @@ struct HttpClient::Impl { } }; -HttpClient::HttpClient(const std::string& host, uint16_t port, AuthHeaders headers, bool enableCompression) : impl_( - std::make_unique(host, port, std::move(headers), enableCompression)) {} +HttpClient::HttpClient(const std::string& host, uint16_t port, AuthHeaders headers, bool enableCompression) + : impl_(std::make_unique(host, port, std::move(headers), enableCompression)) +{ +} HttpClient::~HttpClient() = default; @@ -76,43 +99,41 @@ LayerTilesRequest::Ptr HttpClient::request(const LayerTilesRequest::Ptr& request } auto reader = std::make_unique( - [this](auto&& mapId, auto&& layerId){return impl_->resolve(mapId, layerId);}, + [this](auto&& mapId, auto&& layerId) { return impl_->resolve(mapId, layerId); }, [request](auto&& result) { request->notifyResult(result); }, impl_->stringPoolProvider_); using namespace nlohmann; - // TODO: Currently, cpp-httplib client-POST does not support async responses. - // Those are only supported by GET. So, currently, this HttpClient - // does not profit from the streaming response. However, erdblick is - // is fully able to process async responses as it uses the browser fetch()-API. - auto tileResponse = impl_->client_.Post( - "/tiles", - impl_->headers_, - json::object({ - {"requests", json::array({request->toJson()})}, - {"stringPoolOffsets", reader->stringPoolCache()->stringPoolOffsets()} - }).dump(), - "application/json"); - - if (tileResponse) { - if (tileResponse->status == 200) { - reader->read(tileResponse->body); - } - else if (tileResponse->status == 400) { + auto body = json::object({ + {"requests", json::array({request->toJson()})}, + {"stringPoolOffsets", reader->stringPoolCache()->stringPoolOffsets()}, + }).dump(); + + auto httpReq = drogon::HttpRequest::newHttpRequest(); + httpReq->setMethod(drogon::Post); + httpReq->setPath("/tiles"); + httpReq->setContentTypeCode(drogon::CT_APPLICATION_JSON); + httpReq->setBody(std::move(body)); + applyHeaders(httpReq, impl_->headers_); + + auto [result, resp] = impl_->client_->sendRequest(httpReq); + if (result == drogon::ReqResult::Ok && resp) { + if (resp->statusCode() == drogon::k200OK) { + reader->read(std::string(resp->body())); + } else if (resp->statusCode() == drogon::k400BadRequest) { request->setStatus(RequestStatus::NoDataSource); - } - else if (tileResponse->status == 403) { + } else if (resp->statusCode() == drogon::k403Forbidden) { request->setStatus(RequestStatus::Unauthorized); + } else { + request->setStatus(RequestStatus::Aborted); } - // TODO if multiple LayerTileRequests are ever sent by this client, - // additionally handle RequestStatus::Aborted. - } - else { + } else { request->setStatus(RequestStatus::Aborted); } return request; } -} +} // namespace mapget + diff --git a/libs/http-service/src/http-service.cpp b/libs/http-service/src/http-service.cpp index 9f09586f..1a4249ea 100644 --- a/libs/http-service/src/http-service.cpp +++ b/libs/http-service/src/http-service.cpp @@ -4,7 +4,9 @@ #include "mapget/log.h" #include "mapget/service/config.h" -#include +#include +#include +#include #include #include @@ -90,11 +92,11 @@ class GzipCompressor z_stream strm_{}; }; -[[nodiscard]] AuthHeaders authHeadersFromRequest(uWS::HttpRequest* req) +[[nodiscard]] AuthHeaders authHeadersFromRequest(const drogon::HttpRequestPtr& req) { AuthHeaders headers; - for (auto const& [k, v] : *req) { - headers.emplace(std::string(k), std::string(v)); + for (auto const& [k, v] : req->headers()) { + headers.emplace(k, v); } return headers; } @@ -147,8 +149,7 @@ struct HttpService::Impl static constexpr auto jsonlMimeType = "application/jsonl"; static constexpr auto anyMimeType = "*/*"; - explicit TilesStreamState(Impl const& impl, uWS::HttpResponse* res, uWS::Loop* loop) - : impl_(impl), res_(res), loop_(loop) + explicit TilesStreamState(Impl const& impl, trantor::EventLoop* loop) : impl_(impl), loop_(loop) { static std::atomic_uint64_t nextRequestId; requestId_ = nextRequestId++; @@ -156,6 +157,20 @@ struct HttpService::Impl [this](auto&& msg, auto&& /*msgType*/) { appendOutgoingUnlocked(msg); }, stringOffsets_); } + void attachStream(drogon::ResponseStreamPtr stream) + { + { + std::lock_guard lock(mutex_); + if (aborted_ || responseEnded_) { + if (stream) + stream->close(); + return; + } + stream_ = std::move(stream); + } + scheduleDrain(); + } + void parseRequestFromJson(nlohmann::json const& requestJson) { std::string mapId = requestJson["mapId"]; @@ -200,6 +215,15 @@ struct HttpService::Impl impl_.self_.abort(req); } } + drogon::ResponseStreamPtr stream; + { + std::lock_guard lock(mutex_); + if (responseEnded_.exchange(true)) + return; + stream = std::move(stream_); + } + if (stream) + stream->close(); } void addResult(TileLayer::Ptr const& result) @@ -251,26 +275,32 @@ struct HttpService::Impl return; auto weak = weak_from_this(); - loop_->defer([weak = std::move(weak)]() mutable { + loop_->queueInLoop([weak = std::move(weak)]() mutable { if (auto self = weak.lock()) { self->drainOnLoop(); } }); } - void drainOnLoop() - { - drainScheduled_ = false; - if (aborted_ || responseEnded_) - return; - - constexpr size_t maxChunk = 64 * 1024; + void drainOnLoop() + { + drainScheduled_ = false; + if (aborted_ || responseEnded_) + return; + + constexpr size_t maxChunk = 64 * 1024; + + for (;;) { + std::string chunk; + bool done = false; + bool needAbort = false; + bool scheduleAgain = false; + drogon::ResponseStreamPtr streamToClose; + { + std::lock_guard lock(mutex_); + if (!stream_) + return; - for (;;) { - std::string chunk; - bool done = false; - { - std::lock_guard lock(mutex_); if (!pending_.empty()) { size_t n = std::min(pending_.size(), maxChunk); chunk.assign(pending_.data(), n); @@ -283,25 +313,36 @@ struct HttpService::Impl } done = allDone_; } - } - if (!chunk.empty()) { - bool ok = res_->write(chunk); - if (!ok) { - // Backpressure: resume in onWritable. - return; - } - continue; + if (!chunk.empty()) { + if (!stream_->send(chunk)) { + needAbort = true; + } else if (!pending_.empty() || allDone_) { + // Keep draining until we sent everything and closed the stream. + scheduleAgain = true; + } + } else if (done) { + responseEnded_ = true; + streamToClose = std::move(stream_); + } + } + + if (needAbort) { + onAborted(); + return; } - if (done) { - responseEnded_ = true; - res_->end(); - impl_.tryMemoryTrim(trimResponseType_); - } - return; - } - } + if (done) { + if (streamToClose) + streamToClose->close(); + impl_.tryMemoryTrim(trimResponseType_); + return; + } + if (scheduleAgain) + scheduleDrain(); + return; + } + } void appendOutgoingUnlocked(std::string_view bytes) { @@ -316,8 +357,7 @@ struct HttpService::Impl } Impl const& impl_; - uWS::HttpResponse* res_; - uWS::Loop* loop_; + trantor::EventLoop* loop_; std::mutex mutex_; uint64_t requestId_ = 0; @@ -326,6 +366,7 @@ struct HttpService::Impl ResponseType trimResponseType_ = ResponseType::Binary; std::string pending_; + drogon::ResponseStreamPtr stream_; std::unique_ptr writer_; std::vector requests_; TileLayerStream::StringPoolOffsetMap stringOffsets_; @@ -366,169 +407,154 @@ struct HttpService::Impl } } - void handleTilesRequest(uWS::HttpResponse* res, uWS::HttpRequest* req) const + void handleTilesRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const { - auto* loop = uWS::Loop::get(); - auto state = std::make_shared(*this, res, loop); + auto state = std::make_shared(*this, drogon::app().getLoop()); - std::string accept = std::string(req->getHeader("accept")); - std::string acceptEncoding = std::string(req->getHeader("accept-encoding")); + const std::string accept = req->getHeader("accept"); + const std::string acceptEncoding = req->getHeader("accept-encoding"); auto clientHeaders = authHeadersFromRequest(req); - res->onAborted([state]() { state->onAborted(); }); - - res->onData([this, - res, - state, - clientHeaders = std::move(clientHeaders), - accept = std::move(accept), - acceptEncoding = std::move(acceptEncoding), - body = std::string()](std::string_view chunk, bool last) mutable { - if (state->aborted_ || state->responseEnded_) - return; - - body.append(chunk.data(), chunk.size()); - if (!last) - return; - - nlohmann::json j; - try { - j = nlohmann::json::parse(body); - } - catch (const std::exception& e) { - state->responseEnded_ = true; - res->writeStatus("400 Bad Request"); - res->writeHeader("Content-Type", "text/plain"); - res->end(std::string("Invalid JSON: ") + e.what()); - return; - } - - auto requestsIt = j.find("requests"); - if (requestsIt == j.end() || !requestsIt->is_array()) { - state->responseEnded_ = true; - res->writeStatus("400 Bad Request"); - res->writeHeader("Content-Type", "text/plain"); - res->end("Missing or invalid 'requests' array"); - return; - } - - log().info("Processing tiles request {}", state->requestId_); - for (auto& requestJson : *requestsIt) { - state->parseRequestFromJson(requestJson); - } + nlohmann::json j; + try { + j = nlohmann::json::parse(std::string(req->body())); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON: ") + e.what()); + callback(resp); + return; + } - if (j.contains("stringPoolOffsets")) { - for (auto& item : j["stringPoolOffsets"].items()) { - state->stringOffsets_[item.key()] = item.value().get(); - } - } + auto requestsIt = j.find("requests"); + if (requestsIt == j.end() || !requestsIt->is_array()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Missing or invalid 'requests' array"); + callback(resp); + return; + } - std::string acceptError; - if (!state->setResponseTypeFromAccept(accept, acceptError)) { - state->responseEnded_ = true; - res->writeStatus("400 Bad Request"); - res->writeHeader("Content-Type", "text/plain"); - res->end(acceptError); - return; - } + log().info("Processing tiles request {}", state->requestId_); + for (auto& requestJson : *requestsIt) { + state->parseRequestFromJson(requestJson); + } - const bool gzip = containsGzip(acceptEncoding); - if (gzip) { - state->enableGzip(); + if (j.contains("stringPoolOffsets")) { + for (auto& item : j["stringPoolOffsets"].items()) { + state->stringOffsets_[item.key()] = item.value().get(); } + } - for (auto& request : state->requests_) { - request->onFeatureLayer([state](auto&& layer) { state->addResult(layer); }); - request->onSourceDataLayer([state](auto&& layer) { state->addResult(layer); }); - request->onDone_ = [state](RequestStatus) { state->onRequestDone(); }; - } + std::string acceptError; + if (!state->setResponseTypeFromAccept(accept, acceptError)) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::move(acceptError)); + callback(resp); + return; + } - auto canProcess = self_.request(state->requests_, clientHeaders); - if (!canProcess) { - state->responseEnded_ = true; - std::vector> requestStatuses{}; - bool anyUnauthorized = false; - for (auto const& r : state->requests_) { - auto status = r->getStatus(); - requestStatuses.emplace_back(static_cast>(status)); - anyUnauthorized |= (status == RequestStatus::Unauthorized); - } - res->writeStatus(anyUnauthorized ? "403 Forbidden" : "400 Bad Request"); - res->writeHeader("Content-Type", "application/json"); - res->end(nlohmann::json::object({{"status", requestStatuses}}).dump()); - return; - } + const bool gzip = containsGzip(acceptEncoding); + if (gzip) { + state->enableGzip(); + } - if (j.contains("clientId")) { - abortRequestsForClientId(j["clientId"].get(), state); - } + for (auto& request : state->requests_) { + request->onFeatureLayer([state](auto&& layer) { state->addResult(layer); }); + request->onSourceDataLayer([state](auto&& layer) { state->addResult(layer); }); + request->onDone_ = [state](RequestStatus) { state->onRequestDone(); }; + } - if (gzip) { - res->writeHeader("Content-Encoding", "gzip"); - } + const auto canProcess = self_.request(state->requests_, clientHeaders); + if (!canProcess) { + std::vector> requestStatuses{}; + bool anyUnauthorized = false; + for (auto const& r : state->requests_) { + auto status = r->getStatus(); + requestStatuses.emplace_back(static_cast>(status)); + anyUnauthorized |= (status == RequestStatus::Unauthorized); + } + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(anyUnauthorized ? drogon::k403Forbidden : drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(nlohmann::json::object({{"status", requestStatuses}}).dump()); + callback(resp); + return; + } - res->writeHeader("Content-Type", state->responseType_); - res->onWritable([state](uintmax_t) { - state->drainOnLoop(); - return !state->responseEnded_.load(); - }); + if (j.contains("clientId")) { + abortRequestsForClientId(j["clientId"].get(), state); + } - state->scheduleDrain(); - }); + auto resp = drogon::HttpResponse::newAsyncStreamResponse( + [state](drogon::ResponseStreamPtr stream) { state->attachStream(std::move(stream)); }, + true); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeString(state->responseType_); + if (gzip) { + resp->addHeader("Content-Encoding", "gzip"); + } + callback(resp); } - void handleAbortRequest(uWS::HttpResponse* res) const + void handleAbortRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const { - auto aborted = std::make_shared(false); - res->onAborted([aborted]() { *aborted = true; }); - - res->onData([this, res, aborted, body = std::string()](std::string_view chunk, bool last) mutable { - if (*aborted) - return; - body.append(chunk.data(), chunk.size()); - if (!last) + try { + auto j = nlohmann::json::parse(std::string(req->body())); + if (j.contains("clientId")) { + abortRequestsForClientId(j["clientId"].get()); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("OK"); + callback(resp); return; - - try { - auto j = nlohmann::json::parse(body); - if (j.contains("clientId")) { - abortRequestsForClientId(j["clientId"].get()); - if (*aborted) - return; - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "text/plain"); - res->end("OK"); - return; - } - - if (*aborted) - return; - res->writeStatus("400 Bad Request"); - res->writeHeader("Content-Type", "text/plain"); - res->end("Missing clientId"); - } - catch (const std::exception& e) { - if (*aborted) - return; - res->writeStatus("400 Bad Request"); - res->writeHeader("Content-Type", "text/plain"); - res->end(std::string("Invalid JSON: ") + e.what()); } - }); + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Missing clientId"); + callback(resp); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON: ") + e.what()); + callback(resp); + } } - void handleSourcesRequest(uWS::HttpResponse* res, uWS::HttpRequest* req) const + void handleSourcesRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const { auto sourcesInfo = nlohmann::json::array(); for (auto& source : self_.info(authHeadersFromRequest(req))) { sourcesInfo.push_back(source.toJson()); } - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "application/json"); - res->end(sourcesInfo.dump()); + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(sourcesInfo.dump()); + callback(resp); } - void handleStatusRequest(uWS::HttpResponse* res) const + void handleStatusRequest( + const drogon::HttpRequestPtr&, + std::function&& callback) const { auto serviceStats = self_.getStatistics(); auto cacheStats = self_.cache()->getStatistics(); @@ -542,92 +568,93 @@ struct HttpService::Impl oss << "
" << cacheStats.dump(4) << "
"; oss << ""; - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "text/html"); - res->end(oss.str()); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_TEXT_HTML); + resp->setBody(oss.str()); + callback(resp); } - void handleLocateRequest(uWS::HttpResponse* res) const + void handleLocateRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const { - auto aborted = std::make_shared(false); - res->onAborted([aborted]() { *aborted = true; }); - - res->onData([this, res, aborted, body = std::string()](std::string_view chunk, bool last) mutable { - if (*aborted) - return; - body.append(chunk.data(), chunk.size()); - if (!last) - return; - - try { - nlohmann::json j = nlohmann::json::parse(body); - auto requestsJson = j["requests"]; - auto allResponsesJson = nlohmann::json::array(); - - for (auto const& locateReqJson : requestsJson) { - LocateRequest locateReq{locateReqJson}; - auto responsesJson = nlohmann::json::array(); - for (auto const& resp : self_.locate(locateReq)) - responsesJson.emplace_back(resp.serialize()); - allResponsesJson.emplace_back(responsesJson); - } - - if (*aborted) - return; - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "application/json"); - res->end(nlohmann::json::object({{"responses", allResponsesJson}}).dump()); - } - catch (const std::exception& e) { - if (*aborted) - return; - res->writeStatus("400 Bad Request"); - res->writeHeader("Content-Type", "text/plain"); - res->end(std::string("Invalid JSON: ") + e.what()); - } - }); + try { + nlohmann::json j = nlohmann::json::parse(std::string(req->body())); + auto requestsJson = j["requests"]; + auto allResponsesJson = nlohmann::json::array(); + + for (auto const& locateReqJson : requestsJson) { + LocateRequest locateReq{locateReqJson}; + auto responsesJson = nlohmann::json::array(); + for (auto const& resp : self_.locate(locateReq)) + responsesJson.emplace_back(resp.serialize()); + allResponsesJson.emplace_back(responsesJson); + } + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(nlohmann::json::object({{"responses", allResponsesJson}}).dump()); + callback(resp); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON: ") + e.what()); + callback(resp); + } } - static bool openConfigFile(std::ifstream& configFile, uWS::HttpResponse* res) + static drogon::HttpResponsePtr openConfigFile(std::ifstream& configFile) { auto configFilePath = DataSourceConfigService::get().getConfigFilePath(); if (!configFilePath.has_value()) { - res->writeStatus("404 Not Found"); - res->writeHeader("Content-Type", "text/plain"); - res->end("The config file path is not set. Check the server configuration."); - return false; + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k404NotFound); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The config file path is not set. Check the server configuration."); + return resp; } std::filesystem::path path = *configFilePath; if (!std::filesystem::exists(path)) { - res->writeStatus("404 Not Found"); - res->writeHeader("Content-Type", "text/plain"); - res->end("The server does not have a config file."); - return false; + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k404NotFound); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The server does not have a config file."); + return resp; } configFile.open(*configFilePath); if (!configFile) { - res->writeStatus("500 Internal Server Error"); - res->writeHeader("Content-Type", "text/plain"); - res->end("Failed to open config file."); - return false; + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Failed to open config file."); + return resp; } - return true; + return nullptr; } - static void handleGetConfigRequest(uWS::HttpResponse* res) + static void handleGetConfigRequest( + const drogon::HttpRequestPtr&, + std::function&& callback) { if (!isGetConfigEndpointEnabled()) { - res->writeStatus("403 Forbidden"); - res->writeHeader("Content-Type", "text/plain"); - res->end("The GET /config endpoint is disabled by the server administrator."); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k403Forbidden); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The GET /config endpoint is disabled by the server administrator."); + callback(resp); return; } std::ifstream configFile; - if (!openConfigFile(configFile, res)) { + if (auto errorResp = openConfigFile(configFile)) { + callback(errorResp); return; } @@ -647,149 +674,145 @@ struct HttpService::Impl combinedJson["model"] = jsonConfig; combinedJson["readOnly"] = !isPostConfigEndpointEnabled(); - res->writeStatus("200 OK"); - res->writeHeader("Content-Type", "application/json"); - res->end(combinedJson.dump(2)); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(combinedJson.dump(2)); + callback(resp); } catch (const std::exception& e) { - res->writeStatus("500 Internal Server Error"); - res->writeHeader("Content-Type", "text/plain"); - res->end(std::string("Error processing config file: ") + e.what()); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Error processing config file: ") + e.what()); + callback(resp); } } - void handlePostConfigRequest(uWS::HttpResponse* res) const + void handlePostConfigRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const { if (!isPostConfigEndpointEnabled()) { - res->writeStatus("403 Forbidden"); - res->writeHeader("Content-Type", "text/plain"); - res->end("The POST /config endpoint is not enabled by the server administrator."); + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k403Forbidden); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The POST /config endpoint is not enabled by the server administrator."); + callback(resp); return; } struct ConfigUpdateState : std::enable_shared_from_this { - uWS::HttpResponse* res = nullptr; - uWS::Loop* loop = nullptr; - std::atomic_bool aborted{false}; + trantor::EventLoop* loop = nullptr; std::atomic_bool done{false}; std::atomic_bool wroteConfig{false}; std::unique_ptr subscription; - std::string body; + std::function callback; }; - auto state = std::make_shared(); - state->res = res; - state->loop = uWS::Loop::get(); - - res->onAborted([state]() { - state->aborted = true; - state->done = true; - state->subscription.reset(); - }); - - res->onData([state](std::string_view chunk, bool last) mutable { - if (state->aborted) - return; - state->body.append(chunk.data(), chunk.size()); - if (!last) - return; + std::ifstream configFile; + if (auto errorResp = openConfigFile(configFile)) { + callback(errorResp); + return; + } - std::ifstream configFile; - if (!Impl::openConfigFile(configFile, state->res)) { - state->done = true; - return; - } + nlohmann::json jsonConfig; + try { + jsonConfig = nlohmann::json::parse(std::string(req->body())); + } + catch (const nlohmann::json::parse_error& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON format: ") + e.what()); + callback(resp); + return; + } - nlohmann::json jsonConfig; - try { - jsonConfig = nlohmann::json::parse(state->body); - } - catch (const nlohmann::json::parse_error& e) { - state->res->writeStatus("400 Bad Request"); - state->res->writeHeader("Content-Type", "text/plain"); - state->res->end(std::string("Invalid JSON format: ") + e.what()); - state->done = true; - return; - } + try { + DataSourceConfigService::get().validateDataSourceConfig(jsonConfig); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Validation failed: ") + e.what()); + callback(resp); + return; + } - try { - DataSourceConfigService::get().validateDataSourceConfig(jsonConfig); - } - catch (const std::exception& e) { - state->res->writeStatus("500 Internal Server Error"); - state->res->writeHeader("Content-Type", "text/plain"); - state->res->end(std::string("Validation failed: ") + e.what()); - state->done = true; - return; - } + auto yamlConfig = YAML::Load(configFile); + std::unordered_map maskedSecrets; + yamlToJson(yamlConfig, true, &maskedSecrets); - auto yamlConfig = YAML::Load(configFile); - std::unordered_map maskedSecrets; - yamlToJson(yamlConfig, true, &maskedSecrets); + for (auto const& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { + if (jsonConfig.contains(key)) + yamlConfig[key] = jsonToYaml(jsonConfig[key], maskedSecrets); + } - for (auto const& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { - if (jsonConfig.contains(key)) - yamlConfig[key] = jsonToYaml(jsonConfig[key], maskedSecrets); - } + auto state = std::make_shared(); + state->loop = drogon::app().getLoop(); + state->callback = std::move(callback); - // Subscribe before writing; ignore any callbacks that happen before we write. - state->subscription = DataSourceConfigService::get().subscribe( - [state](std::vector const&) mutable { - if (!state->wroteConfig) { - return; - } - if (state->done.exchange(true) || state->aborted) - return; - state->loop->defer([state]() mutable { - if (state->aborted) - return; - state->res->writeStatus("200 OK"); - state->res->writeHeader("Content-Type", "text/plain"); - state->res->end("Configuration updated and applied successfully."); - state->subscription.reset(); - }); - }, - [state](std::string const& error) mutable { - if (!state->wroteConfig) { - return; - } - if (state->done.exchange(true) || state->aborted) - return; - state->loop->defer([state, error]() mutable { - if (state->aborted) - return; - state->res->writeStatus("500 Internal Server Error"); - state->res->writeHeader("Content-Type", "text/plain"); - state->res->end(std::string("Error applying the configuration: ") + error); - state->subscription.reset(); - }); + // Subscribe before writing; ignore any callbacks that happen before we write. + state->subscription = DataSourceConfigService::get().subscribe( + [state](std::vector const&) mutable { + if (!state->wroteConfig) { + return; + } + if (state->done.exchange(true)) + return; + state->loop->queueInLoop([state]() mutable { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Configuration updated and applied successfully."); + state->callback(resp); + state->subscription.reset(); }); + }, + [state](std::string const& error) mutable { + if (!state->wroteConfig) { + return; + } + if (state->done.exchange(true)) + return; + state->loop->queueInLoop([state, error]() mutable { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Error applying the configuration: ") + error); + state->callback(resp); + state->subscription.reset(); + }); + }); - configFile.close(); - log().trace("Writing new config."); - state->wroteConfig = true; - std::ofstream newConfigFile(*DataSourceConfigService::get().getConfigFilePath()); + configFile.close(); + log().trace("Writing new config."); + state->wroteConfig = true; + if (auto configFilePath = DataSourceConfigService::get().getConfigFilePath()) { + std::ofstream newConfigFile(*configFilePath); newConfigFile << yamlConfig; newConfigFile.close(); + } - // Timeout fail-safe (rare endpoint; ok to spawn a thread). - std::thread([weak = state->weak_from_this()]() { - std::this_thread::sleep_for(std::chrono::seconds(60)); - if (auto state = weak.lock()) { - if (state->done.exchange(true) || state->aborted) - return; - state->loop->defer([state]() mutable { - if (state->aborted) - return; - state->res->writeStatus("500 Internal Server Error"); - state->res->writeHeader("Content-Type", "text/plain"); - state->res->end("Timeout while waiting for config to update."); - state->subscription.reset(); - }); - } - }).detach(); - }); + // Timeout fail-safe (rare endpoint; ok to spawn a thread). + std::thread([weak = state->weak_from_this()]() { + std::this_thread::sleep_for(std::chrono::seconds(60)); + if (auto state = weak.lock()) { + if (state->done.exchange(true)) + return; + state->loop->queueInLoop([state]() mutable { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Timeout while waiting for config to update."); + state->callback(resp); + state->subscription.reset(); + }); + } + }).detach(); } }; @@ -800,15 +823,62 @@ HttpService::HttpService(Cache::Ptr cache, const HttpServiceConfig& config) HttpService::~HttpService() = default; -void HttpService::setup(uWS::App& app) +void HttpService::setup(drogon::HttpAppFramework& app) { - app.post("/tiles", [this](auto* res, auto* req) { impl_->handleTilesRequest(res, req); }); - app.post("/abort", [this](auto* res, auto* /*req*/) { impl_->handleAbortRequest(res); }); - app.get("/sources", [this](auto* res, auto* req) { impl_->handleSourcesRequest(res, req); }); - app.get("/status", [this](auto* res, auto* /*req*/) { impl_->handleStatusRequest(res); }); - app.post("/locate", [this](auto* res, auto* /*req*/) { impl_->handleLocateRequest(res); }); - app.get("/config", [](auto* res, auto* /*req*/) { Impl::handleGetConfigRequest(res); }); - app.post("/config", [this](auto* res, auto* /*req*/) { impl_->handlePostConfigRequest(res); }); + app.registerHandler( + "/tiles", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + impl_->handleTilesRequest(req, std::move(callback)); + }, + {drogon::Post}); + + app.registerHandler( + "/abort", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + impl_->handleAbortRequest(req, std::move(callback)); + }, + {drogon::Post}); + + app.registerHandler( + "/sources", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + impl_->handleSourcesRequest(req, std::move(callback)); + }, + {drogon::Get}); + + app.registerHandler( + "/status", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + impl_->handleStatusRequest(req, std::move(callback)); + }, + {drogon::Get}); + + app.registerHandler( + "/locate", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + impl_->handleLocateRequest(req, std::move(callback)); + }, + {drogon::Post}); + + app.registerHandler( + "/config", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + if (req->method() == drogon::Get) { + Impl::handleGetConfigRequest(req, std::move(callback)); + return; + } + if (req->method() == drogon::Post) { + impl_->handlePostConfigRequest(req, std::move(callback)); + return; + } + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k405MethodNotAllowed); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Method not allowed"); + callback(resp); + }, + {drogon::Get, drogon::Post}); } } // namespace mapget diff --git a/libs/pymapget/CMakeLists.txt b/libs/pymapget/CMakeLists.txt index 8d9a2016..fb7b8066 100644 --- a/libs/pymapget/CMakeLists.txt +++ b/libs/pymapget/CMakeLists.txt @@ -28,17 +28,7 @@ target_compile_features(pymapget INTERFACE cxx_std_17) -FetchContent_GetProperties(cpp-httplib) -if (MSVC AND CPP-HTTPLIB_POPULATED) - # Required because cpp-httplib speaks https via OpenSSL. - # Only needed if httplib came via FetchContent. - set(DEPLOY_FILES - "${OPENSSL_INCLUDE_DIR}/../libcrypto-1_1-x64.dll" - "${OPENSSL_INCLUDE_DIR}/../libssl-1_1-x64.dll" - "${CMAKE_CURRENT_LIST_DIR}/__main__.py") -else() - set(DEPLOY_FILES "${CMAKE_CURRENT_LIST_DIR}/__main__.py") -endif() +set(DEPLOY_FILES "${CMAKE_CURRENT_LIST_DIR}/__main__.py") add_wheel(pymapget NAME mapget diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 8a2deff4..af35f5cb 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -1,6 +1,7 @@ project(test.mapget.unit CXX) add_executable(test.mapget + test-main.cpp test-model.cpp test-model-geometry.cpp test-simfil-geometry.cpp @@ -16,6 +17,9 @@ add_executable(test.mapget add_executable(test.mapget.filelog test-file-logging.cpp) +add_executable(test.mapget.datasource-server + test-datasource-server.cpp) + target_link_libraries(test.mapget PUBLIC mapget-log @@ -23,7 +27,16 @@ target_link_libraries(test.mapget mapget-http-datasource mapget-http-service geojsonsource - Catch2::Catch2WithMain) + Catch2::Catch2) + +target_link_libraries(test.mapget.datasource-server + PUBLIC + mapget-log + mapget-http-datasource) + +target_compile_definitions(test.mapget + PRIVATE + MAPGET_TEST_DATASOURCE_SERVER_EXE=\"$\") target_link_libraries(test.mapget.filelog PUBLIC diff --git a/test/unit/test-datasource-server.cpp b/test/unit/test-datasource-server.cpp new file mode 100644 index 00000000..e5ebfc4f --- /dev/null +++ b/test/unit/test-datasource-server.cpp @@ -0,0 +1,70 @@ +#include "mapget/http-datasource/datasource-server.h" +#include "mapget/log.h" + +#include "nlohmann/json.hpp" + +using namespace mapget; +using namespace nlohmann; + +int main() +{ + setLogLevel("trace", log()); + + auto info = DataSourceInfo::fromJson(R"( + { + "nodeId": "test-datasource", + "mapId": "Tropico", + "layers": { + "WayLayer": { + "featureTypes": + [ + { + "name": "Way", + "uniqueIdCompositions": + [ + [ + { + "partId": "areaId", + "description": "String which identifies the map area.", + "datatype": "STR" + }, + { + "partId": "wayId", + "description": "Globally Unique 32b integer.", + "datatype": "U32" + } + ] + ] + } + ] + }, + "SourceData-WayLayer": { + "type": "SourceData" + } + } + } + )"_json); + + DataSourceServer ds(info); + ds.onTileFeatureRequest( + [&](const auto& tile) + { + auto f = tile->newFeature("Way", {{"areaId", "Area42"}, {"wayId", 0}}); + auto g = f->geom()->newGeometry(GeomType::Line); + g->append({42., 11}); + g->append({42., 12}); + }); + ds.onTileSourceDataRequest([&](const auto&) {}); + ds.onLocateRequest( + [&](LocateRequest const& request) -> std::vector + { + LocateResponse response(request); + response.tileKey_.layerId_ = "WayLayer"; + response.tileKey_.tileId_.value_ = 1; + return {response}; + }); + + ds.go("127.0.0.1", 0, 5000); + ds.waitForSignal(); + return 0; +} diff --git a/test/unit/test-http-datasource.cpp b/test/unit/test-http-datasource.cpp index 8f58c08b..a39f63c6 100644 --- a/test/unit/test-http-datasource.cpp +++ b/test/unit/test-http-datasource.cpp @@ -1,34 +1,167 @@ #include + +#include +#include #include +#include +#include +#include +#include +#include +#include #include -#include +#include + #ifndef _WIN32 -#include #include +#include #endif -#include "httplib.h" -#include "mapget/log.h" -#include "nlohmann/json.hpp" -#include "utility.h" +#include +#include +#include + +#include "process.hpp" + #include "mapget/http-datasource/datasource-client.h" -#include "mapget/http-datasource/datasource-server.h" +#include "mapget/http-service/cli.h" #include "mapget/http-service/http-client.h" #include "mapget/http-service/http-service.h" +#include "mapget/log.h" +#include "mapget/model/info.h" #include "mapget/model/stream.h" #include "mapget/service/config.h" -#include "mapget/http-service/cli.h" + +#include "nlohmann/json.hpp" + +#include "test-http-service-fixture.h" +#include "utility.h" using namespace mapget; namespace fs = std::filesystem; -TEST_CASE("HttpDataSource", "[HttpDataSource]") +namespace { - setLogLevel("trace", log()); - // Create DataSourceInfo. - auto info = DataSourceInfo::fromJson(R"( +class SyncHttpClient +{ +public: + SyncHttpClient(std::string host, uint16_t port) + { + loopThread_ = std::make_unique("MapgetTestHttpClient"); + loopThread_->run(); + + client_ = drogon::HttpClient::newHttpClient( + fmt::format("http://{}:{}/", host, port), + loopThread_->getLoop()); + } + + std::pair get(std::string path) + { + auto req = drogon::HttpRequest::newHttpRequest(); + req->setMethod(drogon::Get); + req->setPath(std::move(path)); + return client_->sendRequest(req); + } + + std::pair postJson(std::string path, std::string body) + { + auto req = drogon::HttpRequest::newHttpRequest(); + req->setMethod(drogon::Post); + req->setPath(std::move(path)); + req->setContentTypeCode(drogon::CT_APPLICATION_JSON); + req->setBody(std::move(body)); + return client_->sendRequest(req); + } + +private: + std::unique_ptr loopThread_; + drogon::HttpClientPtr client_; +}; + +class ChildProcessWithPort +{ +public: + explicit ChildProcessWithPort(std::string exePath) + { + auto stderrCallback = [](const char* bytes, size_t n) { + auto output = std::string(bytes, n); + output.erase(output.find_last_not_of(" \n\r\t") + 1); + if (!output.empty()) + std::cerr << output << std::endl; + }; + + auto stdoutCallback = [this](const char* bytes, size_t n) { + std::lock_guard lock(mutex_); + stdoutBuffer_.append(bytes, n); + + for (;;) { + auto nl = stdoutBuffer_.find_first_of("\r\n"); + if (nl == std::string::npos) + break; + + auto line = stdoutBuffer_.substr(0, nl); + stdoutBuffer_.erase(0, nl + 1); + line.erase(line.find_last_not_of(" \n\r\t") + 1); + + if (!portReady_) { + std::regex portRegex(R"(Running on port (\d+))"); + std::smatch matches; + if (std::regex_search(line, matches, portRegex) && matches.size() > 1) { + port_ = static_cast(std::stoi(matches.str(1))); + portReady_ = true; + cv_.notify_all(); + } + } + } + }; + + process_ = std::make_unique( + fmt::format("\"{}\"", exePath), + "", + stdoutCallback, + stderrCallback, + true); + + std::unique_lock lock(mutex_); +#if defined(NDEBUG) + if (!cv_.wait_for(lock, std::chrono::seconds(10), [this] { return portReady_; })) { + raise("Timeout waiting for the child process to start listening."); + } +#else + log().warn("Using Debug build: will wait forever!"); + cv_.wait(lock, [this] { return portReady_; }); +#endif + } + + ~ChildProcessWithPort() { + if (process_) { + process_->kill(true); + process_->get_exit_status(); + } + } + + [[nodiscard]] uint16_t port() const + { + return port_; + } + +private: + std::unique_ptr process_; + mutable std::mutex mutex_; + std::condition_variable cv_; + std::string stdoutBuffer_; + uint16_t port_ = 0; + bool portReady_ = false; +}; + +nlohmann::json testDataSourceInfoJson() +{ + using nlohmann::json; + return json::parse(R"( + { + "nodeId": "test-datasource", "mapId": "Tropico", "layers": { "WayLayer": { @@ -59,69 +192,41 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") } } } - )"_json); - - // Initialize a DataSource. - DataSourceServer ds(info); - std::atomic_uint32_t dataSourceFeatureRequestCount = 0; - std::atomic_uint32_t dataSourceSourceDataRequestCount = 0; - ds.onTileFeatureRequest( - [&](const auto& tile) - { - auto f = tile->newFeature("Way", {{"areaId", "Area42"}, {"wayId", 0}}); - auto g = f->geom()->newGeometry(GeomType::Line); - g->append({42., 11}); - g->append({42., 12}); - ++dataSourceFeatureRequestCount; - }); - ds.onTileSourceDataRequest( - [&](const auto& tile) { - ++dataSourceSourceDataRequestCount; - }); - ds.onLocateRequest( - [&](LocateRequest const& request) -> std::vector - { - REQUIRE(request.mapId_ == "Tropico"); - REQUIRE(request.typeId_ == "Way"); - REQUIRE(request.featureId_ == KeyValuePairs{{"wayId", 0}}); + )"); +} - LocateResponse response(request); - response.tileKey_.layerId_ = "WayLayer"; - response.tileKey_.tileId_.value_ = 1; - return {response}; - }); +} // namespace - // Launch the DataSource on a separate thread. - ds.go(); +TEST_CASE("HttpDataSource", "[HttpDataSource]") +{ + setLogLevel("trace", log()); + + // Start datasource server in a separate process (Drogon is singleton). + ChildProcessWithPort dsProc(MAPGET_TEST_DATASOURCE_SERVER_EXE); + + // Expected datasource info. + auto info = DataSourceInfo::fromJson(testDataSourceInfoJson()); - // Ensure the DataSource is running. - REQUIRE(ds.isRunning() == true); + SyncHttpClient dsClient("127.0.0.1", dsProc.port()); - SECTION("Fetch /info") + // Fetch /info { - // Initialize an httplib client. - httplib::Client cli("localhost", ds.port()); + auto [result, resp] = dsClient.get("/info"); + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(resp != nullptr); + REQUIRE(resp->statusCode() == drogon::k200OK); - // Send a GET info request. - auto fetchedInfoJson = cli.Get("/info"); - auto fetchedInfo = - DataSourceInfo::fromJson(nlohmann::json::parse(fetchedInfoJson->body)); + auto fetchedInfo = DataSourceInfo::fromJson(nlohmann::json::parse(std::string(resp->body()))); REQUIRE(fetchedInfo.toJson() == info.toJson()); } - SECTION("Fetch /tile") + // Fetch /tile { - // Initialize an httplib client. - httplib::Client cli("localhost", ds.port()); + auto [result, resp] = dsClient.get("/tile?layer=WayLayer&tileId=1"); + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(resp != nullptr); + REQUIRE(resp->statusCode() == drogon::k200OK); - // Send a GET tile request. - auto tileResponse = cli.Get("/tile?layer=WayLayer&tileId=1"); - - // Check that the response is OK. - REQUIRE(tileResponse != nullptr); - REQUIRE(tileResponse->status == 200); - - // Check the response body for expected content. auto receivedTileCount = 0; TileLayerStream::Reader reader( [&](auto&& mapId, auto&& layerId) @@ -133,24 +238,18 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") REQUIRE(tile->id().layer_ == LayerType::Features); receivedTileCount++; }); - reader.read(tileResponse->body); + reader.read(std::string(resp->body())); REQUIRE(receivedTileCount == 1); } - SECTION("Fetch /tile SourceData") + // Fetch /tile SourceData { - // Initialize an httplib client. - httplib::Client cli("localhost", ds.port()); - - // Send a GET tile request - auto tileResponse = cli.Get("/tile?layer=SourceData-WayLayer&tileId=1"); + auto [result, resp] = dsClient.get("/tile?layer=SourceData-WayLayer&tileId=1"); + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(resp != nullptr); + REQUIRE(resp->statusCode() == drogon::k200OK); - // Check that the response is OK. - REQUIRE(tileResponse != nullptr); - REQUIRE(tileResponse->status == 200); - - // Check the response body for expected content. auto receivedTileCount = 0; TileLayerStream::Reader reader( [&](auto&& mapId, auto&& layerId) @@ -162,57 +261,49 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") REQUIRE(tile->id().layer_ == LayerType::SourceData); receivedTileCount++; }); - reader.read(tileResponse->body); + reader.read(std::string(resp->body())); REQUIRE(receivedTileCount == 1); } - SECTION("Fetch /locate") + // Fetch /locate { - // Initialize an httplib client. - httplib::Client cli("localhost", ds.port()); - - // Send a POST locate request. - auto response = cli.Post("/locate", R"({ - "mapId": "Tropico", - "typeId": "Way", - "featureId": ["wayId", 0] - })", "application/json"); - - // Check that the response is OK. - REQUIRE(response != nullptr); - REQUIRE(response->status == 200); - - // Check the response body for expected content. - LocateResponse responseParsed(nlohmann::json::parse(response->body)[0]); + auto [result, resp] = dsClient.postJson( + "/locate", + R"({ + "mapId": "Tropico", + "typeId": "Way", + "featureId": ["wayId", 0] + })"); + + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(resp != nullptr); + REQUIRE(resp->statusCode() == drogon::k200OK); + + LocateResponse responseParsed(nlohmann::json::parse(std::string(resp->body()))[0]); REQUIRE(responseParsed.tileKey_.mapId_ == "Tropico"); REQUIRE(responseParsed.tileKey_.layer_ == LayerType::Features); REQUIRE(responseParsed.tileKey_.layerId_ == "WayLayer"); REQUIRE(responseParsed.tileKey_.tileId_.value_ == 1); } - SECTION("Query mapget HTTP service") + // Query mapget HTTP service (in-process, started once for entire test binary) { + auto& service = test::httpService(); + auto remoteDataSource = std::make_shared("127.0.0.1", dsProc.port()); + service.add(remoteDataSource); + auto countReceivedTiles = [](auto& client, auto mapId, auto layerId, auto tiles) { auto tileCount = 0; - auto request = std::make_shared(mapId, layerId, tiles); - request->onFeatureLayer([&](auto&& tile) { tileCount++; }); - //request->onSourceDataLayer([&](auto&& tile) { tileCount++; }); - + request->onFeatureLayer([&](auto&&) { tileCount++; }); client.request(request)->wait(); return std::make_tuple(request, tileCount); }; - HttpService service; - auto remoteDataSource = std::make_shared("localhost", ds.port()); - service.add(remoteDataSource); - - service.go(); - - SECTION("Query through mapget HTTP service") + // Query through mapget HTTP service { - HttpClient client("localhost", service.port()); + HttpClient client("127.0.0.1", service.port()); auto [request, receivedTileCount] = countReceivedTiles( client, @@ -221,54 +312,47 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") std::vector{{1234, 5678, 9112, 1234}}); REQUIRE(receivedTileCount == 4); - // One tile requested twice, so the cache was used. - REQUIRE(dataSourceFeatureRequestCount == 3); + REQUIRE(request->getStatus() == RequestStatus::Success); } - SECTION("Trigger 400 responses") + // Trigger 400 responses { - HttpClient client("localhost", service.port()); + HttpClient client("127.0.0.1", service.port()); { - auto [request, receivedTileCount] = countReceivedTiles( - client, - "UnknownMap", - "WayLayer", - std::vector{{1234}}); + auto [request, receivedTileCount] = + countReceivedTiles(client, "UnknownMap", "WayLayer", std::vector{{1234}}); REQUIRE(request->getStatus() == RequestStatus::NoDataSource); REQUIRE(receivedTileCount == 0); } { - auto [request, receivedTileCount] = countReceivedTiles( - client, - "Tropico", - "UnknownLayer", - std::vector{{1234}}); + auto [request, receivedTileCount] = + countReceivedTiles(client, "Tropico", "UnknownLayer", std::vector{{1234}}); REQUIRE(request->getStatus() == RequestStatus::NoDataSource); REQUIRE(receivedTileCount == 0); } } - SECTION("Run /locate through service") + // Run /locate through service { - httplib::Client client("localhost", service.port()); - - // Send a POST locate request. - auto response = client.Post("/locate", R"({ - "requests": [{ - "mapId": "Tropico", - "typeId": "Way", - "featureId": ["wayId", 0] - }] - })", "application/json"); - - // Check that the response is OK. - REQUIRE(response != nullptr); - REQUIRE(response->status == 200); - - // Check the response body for expected content. - auto responseJsonLists = nlohmann::json::parse(response->body)["responses"]; + SyncHttpClient serviceClient("127.0.0.1", service.port()); + + auto [result, resp] = serviceClient.postJson( + "/locate", + R"({ + "requests": [{ + "mapId": "Tropico", + "typeId": "Way", + "featureId": ["wayId", 0] + }] + })"); + + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(resp != nullptr); + REQUIRE(resp->statusCode() == drogon::k200OK); + + auto responseJsonLists = nlohmann::json::parse(std::string(resp->body()))["responses"]; REQUIRE(responseJsonLists.size() == 1); auto responseJsonList = responseJsonLists[0]; REQUIRE(responseJsonList.size() == 1); @@ -279,79 +363,52 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") REQUIRE(responseParsed.tileKey_.tileId_.value_ == 1); } - SECTION("Test auth header requirement") + // Test auth header requirement { - remoteDataSource->requireAuthHeaderRegexMatchOption( - "X-USER-ROLE", - std::regex("\\bTropico-Viewer\\b")); + remoteDataSource->requireAuthHeaderRegexMatchOption("X-USER-ROLE", std::regex("\\bTropico-Viewer\\b")); - HttpClient badClient("localhost", service.port()); - HttpClient goodClient("localhost", service.port(), {{"X-USER-ROLE", "Tropico-Viewer"}}); + HttpClient badClient("127.0.0.1", service.port()); + HttpClient goodClient("127.0.0.1", service.port(), {{"X-USER-ROLE", "Tropico-Viewer"}}); - // Check sources REQUIRE(badClient.sources().empty()); REQUIRE(goodClient.sources().size() == 1); - // Try to load tiles with bad client { - auto [request, receivedTileCount] = countReceivedTiles( - badClient, - "Tropico", - "WayLayer", - std::vector{{1234}}); + auto [request, receivedTileCount] = + countReceivedTiles(badClient, "Tropico", "WayLayer", std::vector{{1234}}); REQUIRE(request->getStatus() == RequestStatus::Unauthorized); REQUIRE(receivedTileCount == 0); } - // Try to load tiles with good client { - auto [request, receivedTileCount] = countReceivedTiles( - goodClient, - "Tropico", - "WayLayer", - std::vector{{1234}}); + auto [request, receivedTileCount] = + countReceivedTiles(goodClient, "Tropico", "WayLayer", std::vector{{1234}}); REQUIRE(request->getStatus() == RequestStatus::Success); REQUIRE(receivedTileCount == 1); } } - service.stop(); - REQUIRE(service.isRunning() == false); - } - - SECTION("Wait for data source") - { - auto waitThread = std::thread([&] { ds.waitForSignal(); }); - ds.stop(); - waitThread.join(); - REQUIRE(ds.isRunning() == false); + service.remove(remoteDataSource); } - - ds.stop(); - REQUIRE(ds.isRunning() == false); } TEST_CASE("Configuration Endpoint Tests", "[Configuration]") { + auto& service = test::httpService(); + REQUIRE(service.isRunning() == true); + + SyncHttpClient cli("127.0.0.1", service.port()); + auto tempDir = fs::temp_directory_path() / test::generateTimestampedDirectoryName("mapget_test_http_config"); fs::create_directory(tempDir); auto tempConfigPath = tempDir / "temp_config.yaml"; - // Setting up the server and client. - HttpService service; - service.go(); - REQUIRE(service.isRunning() == true); - httplib::Client cli("localhost", service.port()); - // Set up the config file. DataSourceConfigService::get().reset(); struct SchemaPatchGuard { - ~SchemaPatchGuard() { - DataSourceConfigService::get().setDataSourceConfigSchemaPatch(nlohmann::json::object()); - } + ~SchemaPatchGuard() { DataSourceConfigService::get().setDataSourceConfigSchemaPatch(nlohmann::json::object()); } } schemaPatchGuard; - // Emulate the CLI-provided config-schema patch so http-settings participates in the auto schema. auto schemaPatch = nlohmann::json::parse(R"( { "properties": { @@ -364,110 +421,127 @@ TEST_CASE("Configuration Endpoint Tests", "[Configuration]") )"); DataSourceConfigService::get().setDataSourceConfigSchemaPatch(schemaPatch); - SECTION("Get Configuration - Config File Not Found") { + SECTION("Get Configuration - Config File Not Found") + { DataSourceConfigService::get().loadConfig(tempConfigPath.string()); - auto res = cli.Get("/config"); + auto [result, res] = cli.get("/config"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 404); - REQUIRE(res->body == "The server does not have a config file."); + REQUIRE(res->statusCode() == drogon::k404NotFound); + REQUIRE(std::string(res->body()) == "The server does not have a config file."); } // Create config file for tests that need it { std::ofstream configFile(tempConfigPath); - configFile << "sources: []\nhttp-settings: [{'password': 'hunter2'}]"; // Update http-settings to an array. + configFile << "sources: []\nhttp-settings: [{'password': 'hunter2'}]"; configFile.flush(); configFile.close(); - - // Ensure file is synced to disk - #ifndef _WIN32 + +#ifndef _WIN32 int fd = open(tempConfigPath.c_str(), O_RDONLY); if (fd != -1) { fsync(fd); close(fd); } - #endif +#endif std::this_thread::sleep_for(std::chrono::milliseconds(100)); } - - // Load the config after file is created + DataSourceConfigService::get().loadConfig(tempConfigPath.string()); - - // Give the config watcher time to detect the file std::this_thread::sleep_for(std::chrono::milliseconds(500)); - SECTION("Get Configuration - Not allowed") { + SECTION("Get Configuration - Not allowed") + { setGetConfigEndpointEnabled(false); - auto res = cli.Get("/config"); + auto [result, res] = cli.get("/config"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 403); + REQUIRE(res->statusCode() == drogon::k403Forbidden); } - SECTION("Get Configuration - No Config File Path Set") { + SECTION("Get Configuration - No Config File Path Set") + { setGetConfigEndpointEnabled(true); - DataSourceConfigService::get().loadConfig(""); // Simulate no config path set. - auto res = cli.Get("/config"); + DataSourceConfigService::get().loadConfig(""); + auto [result, res] = cli.get("/config"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 404); - REQUIRE(res->body == "The config file path is not set. Check the server configuration."); + REQUIRE(res->statusCode() == drogon::k404NotFound); + REQUIRE(std::string(res->body()) == + "The config file path is not set. Check the server configuration."); } - SECTION("Get Configuration - Success") { - auto res = cli.Get("/config"); + SECTION("Get Configuration - Success") + { + setGetConfigEndpointEnabled(true); + auto [result, res] = cli.get("/config"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 200); - REQUIRE(res->body.find("sources") != std::string::npos); - REQUIRE(res->body.find("http-settings") != std::string::npos); - - // Ensure that the password is masked as SHA256. - REQUIRE(res->body.find("hunter2") == std::string::npos); - REQUIRE(res->body.find("MASKED:0:f52fbd32b2b3b86ff88ef6c490628285f482af15ddcb29541f94bcf526a3f6c7") != std::string::npos); + REQUIRE(res->statusCode() == drogon::k200OK); + + auto body = std::string(res->body()); + REQUIRE(body.find("sources") != std::string::npos); + REQUIRE(body.find("http-settings") != std::string::npos); + REQUIRE(body.find("hunter2") == std::string::npos); + REQUIRE( + body.find("MASKED:0:f52fbd32b2b3b86ff88ef6c490628285f482af15ddcb29541f94bcf526a3f6c7") != + std::string::npos); } - SECTION("Post Configuration - Not Enabled") { + SECTION("Post Configuration - Not Enabled") + { setPostConfigEndpointEnabled(false); - auto res = cli.Post("/config", "", "application/json"); + auto [result, res] = cli.postJson("/config", ""); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 403); + REQUIRE(res->statusCode() == drogon::k403Forbidden); } - SECTION("Post Configuration - Invalid JSON Format") { + SECTION("Post Configuration - Invalid JSON Format") + { setPostConfigEndpointEnabled(true); - std::string invalidJson = "this is not valid json"; - auto res = cli.Post("/config", invalidJson, "application/json"); + auto [result, res] = cli.postJson("/config", "this is not valid json"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 400); - REQUIRE(res->body.find("Invalid JSON format") != std::string::npos); + REQUIRE(res->statusCode() == drogon::k400BadRequest); + REQUIRE(std::string(res->body()).find("Invalid JSON format") != std::string::npos); } - SECTION("Post Configuration - Missing Sources") { - std::string newConfig = R"({"http-settings": []})"; - auto res = cli.Post("/config", newConfig, "application/json"); + SECTION("Post Configuration - Missing Sources") + { + setPostConfigEndpointEnabled(true); + auto [result, res] = cli.postJson("/config", R"({"http-settings": []})"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 500); - REQUIRE(res->body.starts_with("Validation failed")); + REQUIRE(res->statusCode() == drogon::k500InternalServerError); + REQUIRE(std::string(res->body()).starts_with("Validation failed")); } - SECTION("Post Configuration - Missing Http Settings") { - std::string newConfig = R"({"sources": []})"; - auto res = cli.Post("/config", newConfig, "application/json"); + SECTION("Post Configuration - Missing Http Settings") + { + setPostConfigEndpointEnabled(true); + auto [result, res] = cli.postJson("/config", R"({"sources": []})"); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 500); - REQUIRE(res->body.starts_with("Validation failed")); + REQUIRE(res->statusCode() == drogon::k500InternalServerError); + REQUIRE(std::string(res->body()).starts_with("Validation failed")); } - SECTION("Post Configuration - Valid JSON Config") { + SECTION("Post Configuration - Valid JSON Config") + { + setPostConfigEndpointEnabled(true); std::string newConfig = R"({ "sources": [{"type": "TestDataSource"}], "http-settings": [{"scope": "https://example.com", "password": "MASKED:0:f52fbd32b2b3b86ff88ef6c490628285f482af15ddcb29541f94bcf526a3f6c7"}] })"; - log().set_level(spdlog::level::trace); - auto res = cli.Post("/config", newConfig, "application/json"); + + auto [result, res] = cli.postJson("/config", newConfig); + REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->status == 200); - REQUIRE(res->body == "Configuration updated and applied successfully."); + REQUIRE(res->statusCode() == drogon::k200OK); + REQUIRE(std::string(res->body()) == "Configuration updated and applied successfully."); - // Check that the password SHA was re-substituted. std::ifstream config(*mapget::DataSourceConfigService::get().getConfigFilePath()); std::stringstream configContentStream; configContentStream << config.rdbuf(); @@ -475,9 +549,5 @@ TEST_CASE("Configuration Endpoint Tests", "[Configuration]") REQUIRE(configContent.find("hunter2") != std::string::npos); } - service.stop(); - REQUIRE(service.isRunning() == false); - - // Clean up the test configuration files. fs::remove(tempConfigPath); } diff --git a/test/unit/test-http-service-fixture.h b/test/unit/test-http-service-fixture.h new file mode 100644 index 00000000..bbb9b0c3 --- /dev/null +++ b/test/unit/test-http-service-fixture.h @@ -0,0 +1,17 @@ +#pragma once + +#include "mapget/http-service/http-service.h" + +namespace mapget::test +{ + +// Starts the HTTP service lazily (on first use) and keeps it alive for the +// lifetime of the test process. `shutdownHttpService()` stops the server and +// joins its server thread to avoid Drogon shutdown issues. +HttpService& httpService(); + +// Safe to call even if the service was never started. +void shutdownHttpService(); + +} // namespace mapget::test + diff --git a/test/unit/test-main.cpp b/test/unit/test-main.cpp new file mode 100644 index 00000000..eb67e32f --- /dev/null +++ b/test/unit/test-main.cpp @@ -0,0 +1,50 @@ +#define CATCH_CONFIG_RUNNER + +#include + +#include + +#include "mapget/log.h" +#include "test-http-service-fixture.h" + +namespace mapget::test +{ +namespace +{ + +std::mutex serviceMutex; +HttpService* servicePtr = nullptr; + +} // namespace + +HttpService& httpService() +{ + std::lock_guard lock(serviceMutex); + + if (!servicePtr) { + // Intentionally leaked to avoid destructor ordering issues at process shutdown. + servicePtr = new HttpService(); + servicePtr->go("127.0.0.1", 0, 5000); + } + + return *servicePtr; +} + +void shutdownHttpService() +{ + std::lock_guard lock(serviceMutex); + if (!servicePtr) + return; + + servicePtr->stop(); +} + +} // namespace mapget::test + +int main(int argc, char* argv[]) +{ + auto result = Catch::Session().run(argc, argv); + mapget::test::shutdownHttpService(); + return result; +} + From 0814ffabbd2c7233c8314ef040ccbed3822b0548 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Fri, 23 Jan 2026 19:26:40 +0100 Subject: [PATCH 09/38] Pick ports for integration tests dynamically. --- test/integration/CMakeLists.txt | 63 ++++++---- .../detect-ports-and-prepare-config-yaml.py | 111 ++++++++++++++++++ test/integration/run-with-ports.bash | 20 ++++ 3 files changed, 169 insertions(+), 25 deletions(-) create mode 100644 test/integration/detect-ports-and-prepare-config-yaml.py create mode 100644 test/integration/run-with-ports.bash diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 6ce5ba32..1af0caf9 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -1,9 +1,7 @@ project(test.mapget.integration CXX) -# TODO: Figure out a way to do this without assuming free ports. -set (MAPGET_SERVER_PORT 61852) -set (DATASOURCE_CPP_PORT 61853) -set (DATASOURCE_PY_PORT 61854) +set(MAPGET_PICK_PORTS_PY "${CMAKE_CURRENT_LIST_DIR}/detect-ports-and-prepare-config-yaml.py") +set(MAPGET_RUN_WITH_PORTS "${CMAKE_CURRENT_LIST_DIR}/run-with-ports.bash") add_wheel_test(test-local-example WORKING_DIRECTORY @@ -20,20 +18,23 @@ if (NOT WITH_COVERAGE) WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" COMMANDS + # Pick ports at test runtime (reduces CI flakiness vs. hard-coded ports) + -f "python ${MAPGET_PICK_PORTS_PY} --out-dir .integration/test-cli-cpp" + # Run Python datasource - -b "python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py ${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-cpp/ports.env python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py $DATASOURCE_PY_PORT" # Run C++ datasource - -b "./cpp-sample-http-datasource ${DATASOURCE_CPP_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-cpp/ports.env ./cpp-sample-http-datasource $DATASOURCE_CPP_PORT" # Run service - -b "./mapget --log-level trace serve -p ${MAPGET_SERVER_PORT} -d 127.0.0.1:${DATASOURCE_CPP_PORT} -d 127.0.0.1:${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-cpp/ports.env ./mapget --log-level trace serve -p $MAPGET_SERVER_PORT -d 127.0.0.1:$DATASOURCE_CPP_PORT -d 127.0.0.1:$DATASOURCE_PY_PORT" # Request from cpp datasource - -f "./mapget --log-level trace fetch -s 127.0.0.1:${MAPGET_SERVER_PORT} -m Tropico -l WayLayer -t 12345" + -f "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-cpp/ports.env ./mapget --log-level trace fetch -s 127.0.0.1:$MAPGET_SERVER_PORT -m Tropico -l WayLayer -t 12345" # Request from py datasource - -f "./mapget --log-level trace fetch -s 127.0.0.1:${MAPGET_SERVER_PORT} -m TestMap -l WayLayer -t 12345" + -f "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-cpp/ports.env ./mapget --log-level trace fetch -s 127.0.0.1:$MAPGET_SERVER_PORT -m TestMap -l WayLayer -t 12345" ) endif () @@ -42,11 +43,14 @@ if (NOT WITH_COVERAGE) WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" COMMANDS + # Pick ports at test runtime (reduces CI flakiness vs. hard-coded ports) + -f "python ${MAPGET_PICK_PORTS_PY} --out-dir .integration/test-cli-datasource-exe" + # Run service with auto-launched python datasource - -b "${CMAKE_CURRENT_LIST_DIR}/mapget-exec-datasource.bash ${MAPGET_SERVER_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-datasource-exe/ports.env ${CMAKE_CURRENT_LIST_DIR}/mapget-exec-datasource.bash $MAPGET_SERVER_PORT" # Request from py datasource - -f "./mapget --log-level trace fetch -s 127.0.0.1:${MAPGET_SERVER_PORT} -m TestMap -l WayLayer -t 12345" + -f "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-datasource-exe/ports.env ./mapget --log-level trace fetch -s 127.0.0.1:$MAPGET_SERVER_PORT -m TestMap -l WayLayer -t 12345" ) endif () @@ -56,17 +60,20 @@ if (NOT WITH_COVERAGE) WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" COMMANDS + # Pick ports at test runtime (reduces CI flakiness vs. hard-coded ports) + -f "python ${MAPGET_PICK_PORTS_PY} --out-dir .integration/test-cli-python" + # Run Python datasource - -b "python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py ${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-python/ports.env python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py $DATASOURCE_PY_PORT" # Run C++ datasource - -b "./cpp-sample-http-datasource ${DATASOURCE_CPP_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-python/ports.env ./cpp-sample-http-datasource $DATASOURCE_CPP_PORT" # Run service - -b "python -m mapget --log-level trace serve -p ${MAPGET_SERVER_PORT} -d 127.0.0.1:${DATASOURCE_CPP_PORT} -d 127.0.0.1:${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-python/ports.env python -m mapget --log-level trace serve -p $MAPGET_SERVER_PORT -d 127.0.0.1:$DATASOURCE_CPP_PORT -d 127.0.0.1:$DATASOURCE_PY_PORT" # Request from py datasource - -f "python -m mapget --log-level trace fetch -s 127.0.0.1:${MAPGET_SERVER_PORT} -m TestMap -l WayLayer -t 12345" + -f "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-cli-python/ports.env python -m mapget --log-level trace fetch -s 127.0.0.1:$MAPGET_SERVER_PORT -m TestMap -l WayLayer -t 12345" ) endif() @@ -76,20 +83,23 @@ if (NOT WITH_COVERAGE) WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" COMMANDS + # Pick ports + write config YAMLs at test runtime. + -f "python ${MAPGET_PICK_PORTS_PY} --out-dir .integration/test-config-cpp" + # Run Python datasource - -b "python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py ${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-config-cpp/ports.env python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py $DATASOURCE_PY_PORT" # Run C++ datasource - -b "./cpp-sample-http-datasource ${DATASOURCE_CPP_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-config-cpp/ports.env ./cpp-sample-http-datasource $DATASOURCE_CPP_PORT" # Run service - -b "./mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-service.yaml serve" + -b "./mapget --config .integration/test-config-cpp/sample-service.yaml serve" # Request from py datasource - -f "./mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-second-datasource.yaml fetch" + -f "./mapget --config .integration/test-config-cpp/sample-second-datasource.yaml fetch" # Request from cpp datasource - -f "./mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-first-datasource.yaml fetch" + -f "./mapget --config .integration/test-config-cpp/sample-first-datasource.yaml fetch" ) endif () @@ -99,19 +109,22 @@ if (NOT WITH_COVERAGE) WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" COMMANDS + # Pick ports + write config YAMLs at test runtime. + -f "python ${MAPGET_PICK_PORTS_PY} --out-dir .integration/test-config-py" + # Run Python datasource - -b "python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py ${DATASOURCE_PY_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-config-py/ports.env python ${CMAKE_CURRENT_LIST_DIR}/../../examples/python/datasource.py $DATASOURCE_PY_PORT" # Run C++ datasource - -b "./cpp-sample-http-datasource ${DATASOURCE_CPP_PORT}" + -b "bash ${MAPGET_RUN_WITH_PORTS} .integration/test-config-py/ports.env ./cpp-sample-http-datasource $DATASOURCE_CPP_PORT" # Run service - -b "python -m mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-service.yaml serve" + -b "python -m mapget --config .integration/test-config-py/sample-service.yaml serve" # Request from py datasource - -f "python -m mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-second-datasource.yaml fetch" + -f "python -m mapget --config .integration/test-config-py/sample-second-datasource.yaml fetch" # Request from cpp datasource - -f "python -m mapget --config ${CMAKE_CURRENT_LIST_DIR}/../../examples/config/sample-first-datasource.yaml fetch" + -f "python -m mapget --config .integration/test-config-py/sample-first-datasource.yaml fetch" ) endif () diff --git a/test/integration/detect-ports-and-prepare-config-yaml.py b/test/integration/detect-ports-and-prepare-config-yaml.py new file mode 100644 index 00000000..0360ccdd --- /dev/null +++ b/test/integration/detect-ports-and-prepare-config-yaml.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import re +import socket +from pathlib import Path + + +def _pick_free_tcp_ports(count: int) -> list[int]: + if count <= 0: + raise ValueError("count must be > 0") + + sockets: list[socket.socket] = [] + ports: list[int] = [] + try: + for _ in range(count): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("127.0.0.1", 0)) + sockets.append(s) + ports.append(int(s.getsockname()[1])) + finally: + for s in sockets: + try: + s.close() + except Exception: + pass + + if len(set(ports)) != len(ports): + raise RuntimeError(f"Port picker returned duplicates: {ports}") + + return ports + + +def _patch_sample_service_yaml(text: str, mapget_port: int, datasource_cpp_port: int, datasource_py_port: int) -> str: + text = re.sub( + r"(?m)^(\s*port:\s*)\d+(\s*)$", + rf"\g<1>{mapget_port}\g<2>", + text, + count=1, + ) + text = text.replace("127.0.0.1:61853", f"127.0.0.1:{datasource_cpp_port}") + text = text.replace("127.0.0.1:61854", f"127.0.0.1:{datasource_py_port}") + return text + + +def _patch_sample_fetch_yaml(text: str, mapget_port: int) -> str: + return re.sub( + r"(?m)^(\s*server:\s*127\.0\.0\.1:)\d+(\s*)$", + rf"\g<1>{mapget_port}\g<2>", + text, + count=1, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--out-dir", required=True, help="Output directory for generated files (created if needed).") + args = parser.parse_args() + + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + # Pick ports at test runtime (reduces collision risk vs. configure-time selection). + mapget_port, datasource_cpp_port, datasource_py_port = _pick_free_tcp_ports(3) + + ports_env = out_dir / "ports.env" + ports_env.write_text( + "\n".join( + [ + f"export MAPGET_SERVER_PORT={mapget_port}", + f"export DATASOURCE_CPP_PORT={datasource_cpp_port}", + f"export DATASOURCE_PY_PORT={datasource_py_port}", + "", + ] + ), + encoding="utf-8", + newline="\n", + ) + + repo_root = Path(__file__).resolve().parents[2] + examples_config = repo_root / "examples" / "config" + + sample_service = (examples_config / "sample-service.yaml").read_text(encoding="utf-8") + (out_dir / "sample-service.yaml").write_text( + _patch_sample_service_yaml(sample_service, mapget_port, datasource_cpp_port, datasource_py_port), + encoding="utf-8", + newline="\n", + ) + + sample_first = (examples_config / "sample-first-datasource.yaml").read_text(encoding="utf-8") + (out_dir / "sample-first-datasource.yaml").write_text( + _patch_sample_fetch_yaml(sample_first, mapget_port), + encoding="utf-8", + newline="\n", + ) + + sample_second = (examples_config / "sample-second-datasource.yaml").read_text(encoding="utf-8") + (out_dir / "sample-second-datasource.yaml").write_text( + _patch_sample_fetch_yaml(sample_second, mapget_port), + encoding="utf-8", + newline="\n", + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/test/integration/run-with-ports.bash b/test/integration/run-with-ports.bash new file mode 100644 index 00000000..b0f17c54 --- /dev/null +++ b/test/integration/run-with-ports.bash @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -lt 2 ]]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +ports_env="$1" +shift + +# shellcheck disable=SC1090 +source "$ports_env" + +# Note: The wheel test harness passes commands as a single string, so `$MAPGET_SERVER_PORT` +# etc. are not expanded. We intentionally use `eval` here so the variables sourced above +# are expanded before executing the command. +eval "exec $*" + From 52c5d1038cf6e355792ad578a00b1166b985ca10 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 26 Jan 2026 13:23:27 +0100 Subject: [PATCH 10/38] Fix tests and sonar findings. --- cmake/deps.cmake | 2 +- libs/http-datasource/src/datasource-server.cpp | 2 +- libs/http-datasource/src/http-server.cpp | 10 +++++++--- libs/http-service/src/http-service.cpp | 3 +++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/cmake/deps.cmake b/cmake/deps.cmake index 629fc978..323dc283 100644 --- a/cmake/deps.cmake +++ b/cmake/deps.cmake @@ -140,7 +140,7 @@ if (MAPGET_WITH_SERVICE OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING) endif() if (MAPGET_WITH_WHEEL AND NOT TARGET python-cmake-wheel) - CPMAddPackage("gh:Klebert-Engineering/python-cmake-wheel@1.2.0") + CPMAddPackage("gh:Klebert-Engineering/python-cmake-wheel#80592e483cc2be044f64e35c4686a00a9126abd2@1.2.1") endif() if (MAPGET_ENABLE_TESTING) diff --git a/libs/http-datasource/src/datasource-server.cpp b/libs/http-datasource/src/datasource-server.cpp index 30f83794..98ea11b0 100644 --- a/libs/http-datasource/src/datasource-server.cpp +++ b/libs/http-datasource/src/datasource-server.cpp @@ -121,7 +121,7 @@ void DataSourceServer::setup(drogon::HttpAppFramework& app) std::string content; TileLayerStream::StringPoolOffsetMap stringPoolOffsets{{impl_->info_.nodeId_, stringPoolOffsetParam}}; TileLayerStream::Writer layerWriter{ - [&](std::string bytes, TileLayerStream::MessageType) { content.append(bytes); }, + [&](std::string const& bytes, TileLayerStream::MessageType) { content.append(bytes); }, stringPoolOffsets}; layerWriter.write(tileLayer); diff --git a/libs/http-datasource/src/http-server.cpp b/libs/http-datasource/src/http-server.cpp index f7106509..bd917360 100644 --- a/libs/http-datasource/src/http-server.cpp +++ b/libs/http-datasource/src/http-server.cpp @@ -261,11 +261,15 @@ bool HttpServer::mountFileSystem(std::string const& pathFromTo) if (ec) return false; - if (!std::filesystem::exists(fsRoot, ec) || ec || !std::filesystem::is_directory(fsRoot, ec) || ec) + auto exists = std::filesystem::exists(fsRoot, ec); + if (!exists || ec) + return false; + auto isDirectory = std::filesystem::is_directory(fsRoot, ec); + if (isDirectory || ec) return false; - std::lock_guard lock(impl_->mountsMutex_); - impl_->mounts_.push_back(MountPoint{std::move(urlPrefix), std::move(fsRoot)}); + std::scoped_lock lock(impl_->mountsMutex_); + impl_->mounts_.emplace_back(MountPoint{std::move(urlPrefix), std::move(fsRoot)}); return true; } diff --git a/libs/http-service/src/http-service.cpp b/libs/http-service/src/http-service.cpp index 1a4249ea..b1e531d5 100644 --- a/libs/http-service/src/http-service.cpp +++ b/libs/http-service/src/http-service.cpp @@ -59,6 +59,9 @@ class GzipCompressor ~GzipCompressor() { deflateEnd(&strm_); } + GzipCompressor(GzipCompressor const&) = delete; + GzipCompressor(GzipCompressor&&) = delete; + std::string compress(const char* data, size_t size, int flush_mode = Z_NO_FLUSH) { std::string result; From b6f5d6ba18b11d2d2dc75d2a0d9476fcfd4dcbea Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 26 Jan 2026 14:14:25 +0100 Subject: [PATCH 11/38] Re-Add TODO. --- libs/http-service/src/http-client.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libs/http-service/src/http-client.cpp b/libs/http-service/src/http-client.cpp index ea6bb04e..e618a649 100644 --- a/libs/http-service/src/http-client.cpp +++ b/libs/http-service/src/http-client.cpp @@ -120,6 +120,10 @@ LayerTilesRequest::Ptr HttpClient::request(const LayerTilesRequest::Ptr& request auto [result, resp] = impl_->client_->sendRequest(httpReq); if (result == drogon::ReqResult::Ok && resp) { if (resp->statusCode() == drogon::k200OK) { + // TODO: Support streamed/chunked tile responses. + // Drogon's `HttpClient` API only provides the full buffered body. + // True streaming would require a custom client built on + // `trantor::TcpClient` (still within the Drogon dependency). reader->read(std::string(resp->body())); } else if (resp->statusCode() == drogon::k400BadRequest) { request->setStatus(RequestStatus::NoDataSource); @@ -136,4 +140,3 @@ LayerTilesRequest::Ptr HttpClient::request(const LayerTilesRequest::Ptr& request } } // namespace mapget - From e6af301bc7b1fb0b465c5bfe8839915854320ee8 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 26 Jan 2026 18:31:35 +0100 Subject: [PATCH 12/38] Implement Websocket protocol for erdblick. --- docs/mapget-api.md | 36 +- docs/mapget-dev-guide.md | 13 +- docs/mapget-user-guide.md | 2 +- libs/http-service/CMakeLists.txt | 8 + libs/http-service/src/config-handler.cpp | 229 +++++ libs/http-service/src/http-service-impl.cpp | 39 + libs/http-service/src/http-service-impl.h | 72 ++ libs/http-service/src/http-service.cpp | 823 +----------------- libs/http-service/src/locate-handler.cpp | 43 + libs/http-service/src/sources-handler.cpp | 27 + libs/http-service/src/status-handler.cpp | 34 + libs/http-service/src/tiles-http-handler.cpp | 420 +++++++++ libs/http-service/src/tiles-ws-controller.cpp | 558 ++++++++++++ libs/http-service/src/tiles-ws-controller.h | 19 + libs/model/include/mapget/model/stream.h | 6 + libs/model/src/stream.cpp | 7 +- test/unit/test-http-datasource.cpp | 293 +++++++ 17 files changed, 1786 insertions(+), 843 deletions(-) create mode 100644 libs/http-service/src/config-handler.cpp create mode 100644 libs/http-service/src/http-service-impl.cpp create mode 100644 libs/http-service/src/http-service-impl.h create mode 100644 libs/http-service/src/locate-handler.cpp create mode 100644 libs/http-service/src/sources-handler.cpp create mode 100644 libs/http-service/src/status-handler.cpp create mode 100644 libs/http-service/src/tiles-http-handler.cpp create mode 100644 libs/http-service/src/tiles-ws-controller.cpp create mode 100644 libs/http-service/src/tiles-ws-controller.h diff --git a/docs/mapget-api.md b/docs/mapget-api.md index e4e7ce11..18dffb33 100644 --- a/docs/mapget-api.md +++ b/docs/mapget-api.md @@ -1,12 +1,12 @@ -# REST/GeoJSON API Guide +# HTTP / WebSocket API Guide -Mapget exposes a small HTTP API that lets clients discover datasources, stream tiles, abort long‑running requests, locate features by ID and inspect or update the running configuration. This guide describes the endpoints and their request and response formats. +Mapget exposes a small HTTP + WebSocket API that lets clients discover datasources, stream tiles, locate features by ID and inspect or update the running configuration. This guide describes the endpoints and their request and response formats. ## Base URL and formats The server started by `mapget serve` listens on the configured host and port (by default on all interfaces and an automatically chosen port). All endpoints are rooted at that host and port. -Requests that send JSON use `Content-Type: application/json`. Tile streaming supports two response encodings, selected via the `Accept` header: +Requests that send JSON use `Content-Type: application/json`. HTTP tile streaming supports two response encodings, selected via the `Accept` header: - `Accept: application/jsonl` returns a JSON‑Lines stream where each line is one JSON object. - `Accept: application/binary` returns a compact binary stream optimized for high-volume traffic. @@ -23,9 +23,9 @@ The binary format and the logical feature model are described in more detail in Each item contains map ID, available layers and basic metadata. This endpoint is typically used by frontends to discover which maps and layers can be requested via `/tiles`. -## `/tiles` – stream tiles +## `/tiles` – stream tiles (HTTP) -`POST /tiles` streams tiles for one or more map–layer combinations. It is the main data retrieval endpoint used by clients such as erdblick. +`POST /tiles` streams tiles for one or more map–layer combinations. - **Method:** `POST` - **Request body (JSON):** @@ -34,7 +34,6 @@ Each item contains map ID, available layers and basic metadata. This endpoint is - `layerId`: string, ID of the layer within that map. - `tileIds`: array of numeric tile IDs in mapget’s tiling scheme. - `stringPoolOffsets` (optional): dictionary from datasource node ID to last known string ID. Used by advanced clients to avoid receiving the same field names repeatedly in the binary stream. - - `clientId` (optional): arbitrary string identifying this client connection for abort handling. - **Response:** - `application/jsonl` if `Accept: application/jsonl` is sent. - `application/binary` if `Accept: application/binary` is sent, using the tile stream protocol. @@ -43,6 +42,21 @@ Tiles are streamed as they become available. In JSONL mode, each line is the JSO If `Accept-Encoding: gzip` is set, the server compresses responses where possible, which is especially useful for JSONL streams. +To cancel an in-flight HTTP stream, close the HTTP connection. + +## `/tiles` – stream tiles (WebSocket) + +`GET /tiles` supports WebSocket upgrades. This is the preferred tile streaming mode for interactive clients because it supports long-lived connections and request replacement without introducing an extra abort endpoint. + +- **Connect:** `ws://:/tiles` +- **Client → Server:** send one *text* message containing the same JSON body as for `POST /tiles` (`requests`, optional `stringPoolOffsets`). + - `stringPoolOffsets` is optional; the server remembers the latest offsets per WebSocket connection. Clients may re-send it to reset/resync offsets. +- **Server → Client:** sends only *binary* WebSocket messages. Each WebSocket message contains exactly one `TileLayerStream` VTLV frame. + - `StringPool`, `TileFeatureLayer`, `TileSourceDataLayer` are unchanged. + - `Status` frames contain UTF-8 JSON payload describing per-request `RequestStatus` transitions and a human-readable message. The final status frame has `"allDone": true`. + +To cancel, either send a new request message on the same connection (which replaces the current one) or close the WebSocket connection. + ### Why JSONL instead of JSON? JSON Lines is better suited to streaming large responses than a single JSON array. Clients can start processing the first tiles immediately, do not need to buffer the complete response in memory, and can naturally consume the stream with incremental parsers. @@ -87,16 +101,6 @@ Each line in the JSONL response is a GeoJSON-like FeatureCollection with additio The `error` object is only present if an error occurred while filling the tile. When present, the `features` array may be empty or contain partial data. -## `/abort` – cancel tile streaming - -`POST /abort` cancels a running `/tiles` request that was started with a matching `clientId`. It is useful when the viewport changes and the previous stream should be abandoned. - -- **Method:** `POST` -- **Request body (JSON):** `{ "clientId": "" }` -- **Response:** `text/plain` confirmation; a 400 status code if `clientId` is missing. - -Internally the service marks the matching tile requests as aborted and stops scheduling further work for them. - ### Curl Call Example For example, the following curl call could be used to stream GeoJSON feature objects diff --git a/docs/mapget-dev-guide.md b/docs/mapget-dev-guide.md index d7c9b905..3d95b117 100644 --- a/docs/mapget-dev-guide.md +++ b/docs/mapget-dev-guide.md @@ -150,7 +150,7 @@ sequenceDiagram participant Ds as DataSource participant Cache - Client->>Http: POST /tiles
requests, clientId + Client->>Http: POST /tiles
requests Http->>Service: request(requests, headers) Service->>Worker: enqueue jobs per datasource loop per tile @@ -169,30 +169,29 @@ sequenceDiagram Service-->>Http: request complete ``` -If a client supplies a `clientId` in the `/tiles` request, the HTTP layer uses it to track open requests and to implement `/abort`. +For interactive clients, tile streaming can also be done via WebSocket `GET /tiles`, where sending a new request message replaces the current in-flight request on that connection. ## HTTP service internals `mapget::HttpService` binds the core service to an HTTP server implementation. Its responsibilities are: -- map HTTP endpoints to service calls (`/sources`, `/tiles`, `/abort`, `/status`, `/locate`, `/config`), +- map HTTP/WebSocket endpoints to service calls (`/sources`, `/tiles`, `/status`, `/locate`, `/config`), - parse JSON requests and build `LayerTilesRequest` objects, - serialize tile responses as JSONL or binary streams, -- manage per‑client state such as `clientId` for abort handling, and - provide `/config` as a JSON view on the YAML config file. ### Tile streaming For `/tiles`, the HTTP layer: -- parses the JSON body to extract `requests`, `stringPoolOffsets` and an optional `clientId`, +- parses the JSON body to extract `requests` and optional `stringPoolOffsets`, - constructs one `LayerTilesRequest` per map–layer combination, - attaches callbacks that feed results into a shared `HttpTilesRequestState`, and - sends out each tile as soon as it is produced by the service. In JSONL mode the response is a sequence of newline‑separated JSON objects. In binary mode the HTTP layer uses `TileLayerStream::Writer` to serialize string pool updates and tile blobs. Binary responses can optionally be compressed using gzip if the client sends `Accept-Encoding: gzip`. -The `/abort` endpoint uses the `clientId` mechanism to cancel all open tile requests for a given client and to prevent further work from being scheduled for them. +WebSocket `/tiles` uses the same request JSON shape but responds with binary VTLV frames only, and includes `Status` frames (JSON payload) whenever a request’s `RequestStatus` changes. ### Configuration endpoints @@ -207,7 +206,7 @@ These endpoints are guarded by command‑line flags: `--no-get-config` disables The model library provides both the binary tile encoding and the simfil query integration: -- `TileLayerStream::Writer` and `TileLayerStream::Reader` handle versioned, type‑tagged messages for string pools and tile layers. Each message starts with a protocol version, a `MessageType` (string pool, feature tile, SourceData tile, end-of-stream), and a payload size. +- `TileLayerStream::Writer` and `TileLayerStream::Reader` handle versioned, type‑tagged messages for string pools and tile layers. Each message starts with a protocol version, a `MessageType` (string pool, feature tile, SourceData tile, status, end-of-stream), and a payload size. - `TileFeatureLayer` derives from `simfil::ModelPool` and exposes methods such as `evaluate(...)` and `complete(...)` to run simfil expressions and obtain completion candidates. String pools are streamed incrementally. The server keeps a `StringPoolOffsetMap` that tracks, for each ongoing tile request, the highest string ID known to a given client per datasource node id. When a tile is written, `TileLayerStream::Writer` compares that offset with the current `StringPool::highest()` value: diff --git a/docs/mapget-user-guide.md b/docs/mapget-user-guide.md index 978cf6c9..68964c29 100644 --- a/docs/mapget-user-guide.md +++ b/docs/mapget-user-guide.md @@ -10,7 +10,7 @@ The guide is split into several focused documents: - [**Setup Guide**](mapget-setup.md) explains how to install mapget via `pip`, how to build the native executable from source, and how to start a server or use the built‑in `fetch` client for quick experiments. - [**Configuration Guide**](mapget-config.md) documents the YAML configuration file used with `--config`, the supported datasource types (`DataSourceHost`, `DataSourceProcess`, `GridDataSource`, `GeoJsonFolder) and the optional `http-settings` section used by tools and UIs. -- [**REST API Guide**](mapget-api.md) describes the HTTP endpoints exposed by `mapget serve`, including `/sources`, `/tiles`, `/abort`, `/status`, `/locate` and `/config`, along with their request and response formats and example calls. +- [**HTTP / WebSocket API Guide**](mapget-api.md) describes the endpoints exposed by `mapget serve`, including `/sources`, `/tiles`, `/status`, `/locate` and `/config`, along with their request and response formats and example calls. - [**Caching Guide**](mapget-cache.md) covers the available cache modes (`memory`, `persistent`, `none`), explains how to configure cache size and location, and shows how to inspect cache statistics via the status endpoint. - [**Simfil Language Extensions**](mapget-simfil-extensions.md) introduces the feature model, tiling scheme, geometry and validity concepts, and the binary tile stream format. This chapter is especially relevant if you are writing datasources or low‑level clients. - [**Layered Data Model**](mapget-model.md) introduces the feature model, tiling scheme, geometry and validity concepts, and the binary tile stream format. This chapter is especially relevant if you are writing datasources or low‑level clients. diff --git a/libs/http-service/CMakeLists.txt b/libs/http-service/CMakeLists.txt index b9e8799f..99476b81 100644 --- a/libs/http-service/CMakeLists.txt +++ b/libs/http-service/CMakeLists.txt @@ -6,7 +6,15 @@ add_library(mapget-http-service STATIC include/mapget/http-service/cli.h src/http-service.cpp + src/http-service-impl.h + src/http-service-impl.cpp + src/config-handler.cpp src/http-client.cpp + src/locate-handler.cpp + src/sources-handler.cpp + src/status-handler.cpp + src/tiles-http-handler.cpp + src/tiles-ws-controller.cpp src/cli.cpp) target_include_directories(mapget-http-service diff --git a/libs/http-service/src/config-handler.cpp b/libs/http-service/src/config-handler.cpp new file mode 100644 index 00000000..82ac86e6 --- /dev/null +++ b/libs/http-service/src/config-handler.cpp @@ -0,0 +1,229 @@ +#include "http-service-impl.h" + +#include "cli.h" +#include "mapget/log.h" +#include "mapget/service/config.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "nlohmann/json.hpp" +#include "yaml-cpp/yaml.h" + +namespace mapget +{ + +drogon::HttpResponsePtr HttpService::Impl::openConfigFile(std::ifstream& configFile) +{ + auto configFilePath = DataSourceConfigService::get().getConfigFilePath(); + if (!configFilePath.has_value()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k404NotFound); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The config file path is not set. Check the server configuration."); + return resp; + } + + std::filesystem::path path = *configFilePath; + if (!std::filesystem::exists(path)) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k404NotFound); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The server does not have a config file."); + return resp; + } + + configFile.open(*configFilePath); + if (!configFile) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Failed to open config file."); + return resp; + } + + return nullptr; +} + +void HttpService::Impl::handleGetConfigRequest( + const drogon::HttpRequestPtr& /*req*/, + std::function&& callback) +{ + if (!isGetConfigEndpointEnabled()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k403Forbidden); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The GET /config endpoint is disabled by the server administrator."); + callback(resp); + return; + } + + std::ifstream configFile; + if (auto errorResp = openConfigFile(configFile)) { + callback(errorResp); + return; + } + + nlohmann::json jsonSchema = DataSourceConfigService::get().getDataSourceConfigSchema(); + + try { + YAML::Node configYaml = YAML::Load(configFile); + nlohmann::json jsonConfig; + std::unordered_map maskedSecretMap; + for (const auto& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { + if (auto configYamlEntry = configYaml[key]) + jsonConfig[key] = yamlToJson(configYaml[key], true, &maskedSecretMap); + } + + nlohmann::json combinedJson; + combinedJson["schema"] = jsonSchema; + combinedJson["model"] = jsonConfig; + combinedJson["readOnly"] = !isPostConfigEndpointEnabled(); + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(combinedJson.dump(2)); + callback(resp); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Error processing config file: ") + e.what()); + callback(resp); + } +} + +void HttpService::Impl::handlePostConfigRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const +{ + if (!isPostConfigEndpointEnabled()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k403Forbidden); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("The POST /config endpoint is not enabled by the server administrator."); + callback(resp); + return; + } + + struct ConfigUpdateState : std::enable_shared_from_this + { + trantor::EventLoop* loop = nullptr; + std::atomic_bool done{false}; + std::atomic_bool wroteConfig{false}; + std::unique_ptr subscription; + std::function callback; + }; + + std::ifstream configFile; + if (auto errorResp = openConfigFile(configFile)) { + callback(errorResp); + return; + } + + nlohmann::json jsonConfig; + try { + jsonConfig = nlohmann::json::parse(std::string(req->body())); + } + catch (const nlohmann::json::parse_error& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON format: ") + e.what()); + callback(resp); + return; + } + + try { + DataSourceConfigService::get().validateDataSourceConfig(jsonConfig); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Validation failed: ") + e.what()); + callback(resp); + return; + } + + auto yamlConfig = YAML::Load(configFile); + std::unordered_map maskedSecrets; + yamlToJson(yamlConfig, true, &maskedSecrets); + + for (auto const& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { + if (jsonConfig.contains(key)) + yamlConfig[key] = jsonToYaml(jsonConfig[key], maskedSecrets); + } + + auto state = std::make_shared(); + state->loop = drogon::app().getLoop(); + state->callback = std::move(callback); + + state->subscription = DataSourceConfigService::get().subscribe( + [state](std::vector const&) mutable { + if (!state->wroteConfig) { + return; + } + if (state->done.exchange(true)) + return; + state->loop->queueInLoop([state]() mutable { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Configuration updated and applied successfully."); + state->callback(resp); + state->subscription.reset(); + }); + }, + [state](std::string const& error) mutable { + if (!state->wroteConfig) { + return; + } + if (state->done.exchange(true)) + return; + state->loop->queueInLoop([state, error]() mutable { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Error applying the configuration: ") + error); + state->callback(resp); + state->subscription.reset(); + }); + }); + + configFile.close(); + log().trace("Writing new config."); + state->wroteConfig = true; + if (auto configFilePath = DataSourceConfigService::get().getConfigFilePath()) { + std::ofstream newConfigFile(*configFilePath); + newConfigFile << yamlConfig; + newConfigFile.close(); + } + + std::thread([weak = state->weak_from_this()]() { + std::this_thread::sleep_for(std::chrono::seconds(60)); + if (auto state = weak.lock()) { + if (state->done.exchange(true)) + return; + state->loop->queueInLoop([state]() mutable { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k500InternalServerError); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Timeout while waiting for config to update."); + state->callback(resp); + state->subscription.reset(); + }); + } + }).detach(); +} + +} // namespace mapget + diff --git a/libs/http-service/src/http-service-impl.cpp b/libs/http-service/src/http-service-impl.cpp new file mode 100644 index 00000000..1f323b31 --- /dev/null +++ b/libs/http-service/src/http-service-impl.cpp @@ -0,0 +1,39 @@ +#include "http-service-impl.h" + +#include "mapget/log.h" + +#ifdef __linux__ +#include +#endif + +namespace mapget +{ + +HttpService::Impl::Impl(HttpService& self, const HttpServiceConfig& config) : self_(self), config_(config) {} + +void HttpService::Impl::tryMemoryTrim(ResponseType responseType) const +{ + uint64_t interval = + (responseType == ResponseType::Binary) ? config_.memoryTrimIntervalBinary : config_.memoryTrimIntervalJson; + + if (interval == 0) { + return; + } + + auto& counter = (responseType == ResponseType::Binary) ? binaryRequestCounter_ : jsonRequestCounter_; + auto count = counter.fetch_add(1, std::memory_order_relaxed); + if ((count % interval) != 0) { + return; + } + +#ifdef __linux__ +#ifndef NDEBUG + const char* typeStr = (responseType == ResponseType::Binary) ? "binary" : "JSON"; + log().debug("Trimming memory after {} {} requests (interval: {})", count, typeStr, interval); +#endif + malloc_trim(0); +#endif +} + +} // namespace mapget + diff --git a/libs/http-service/src/http-service-impl.h b/libs/http-service/src/http-service-impl.h new file mode 100644 index 00000000..2db77a6c --- /dev/null +++ b/libs/http-service/src/http-service-impl.h @@ -0,0 +1,72 @@ +#pragma once + +#include "http-service.h" + +#include +#include + +#include +#include +#include +#include + +namespace mapget +{ + +namespace detail +{ + +[[nodiscard]] inline AuthHeaders authHeadersFromRequest(const drogon::HttpRequestPtr& req) +{ + AuthHeaders headers; + for (auto const& [k, v] : req->headers()) { + headers.emplace(k, v); + } + return headers; +} + +} // namespace detail + +struct HttpService::Impl +{ + HttpService& self_; + HttpServiceConfig config_; + mutable std::atomic binaryRequestCounter_{0}; + mutable std::atomic jsonRequestCounter_{0}; + + explicit Impl(HttpService& self, const HttpServiceConfig& config); + + enum class ResponseType { Binary, Json }; + + void tryMemoryTrim(ResponseType responseType) const; + + struct TilesStreamState; + + void handleTilesRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const; + + void handleSourcesRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const; + + void handleStatusRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const; + + void handleLocateRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const; + + static drogon::HttpResponsePtr openConfigFile(std::ifstream& configFile); + + static void handleGetConfigRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback); + + void handlePostConfigRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const; +}; + +} // namespace mapget diff --git a/libs/http-service/src/http-service.cpp b/libs/http-service/src/http-service.cpp index b1e531d5..4e377448 100644 --- a/libs/http-service/src/http-service.cpp +++ b/libs/http-service/src/http-service.cpp @@ -1,824 +1,16 @@ -#include "http-service.h" +#include "http-service-impl.h" -#include "cli.h" -#include "mapget/log.h" -#include "mapget/service/config.h" +#include "tiles-ws-controller.h" #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "nlohmann/json-schema.hpp" -#include "nlohmann/json.hpp" -#include "yaml-cpp/yaml.h" - -#include - -#ifdef __linux__ -#include -#endif +#include namespace mapget { -namespace -{ - -/** - * Simple gzip compressor for streaming compression. - */ -class GzipCompressor -{ -public: - GzipCompressor() - { - strm_.zalloc = Z_NULL; - strm_.zfree = Z_NULL; - strm_.opaque = Z_NULL; - // 16+MAX_WBITS enables gzip format (not just deflate) - int ret = deflateInit2( - &strm_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 16 + MAX_WBITS, 8, Z_DEFAULT_STRATEGY); - if (ret != Z_OK) { - throw std::runtime_error("Failed to initialize gzip compressor"); - } - } - - ~GzipCompressor() { deflateEnd(&strm_); } - - GzipCompressor(GzipCompressor const&) = delete; - GzipCompressor(GzipCompressor&&) = delete; - - std::string compress(const char* data, size_t size, int flush_mode = Z_NO_FLUSH) - { - std::string result; - if (size == 0 && flush_mode == Z_NO_FLUSH) { - return result; - } - - strm_.avail_in = static_cast(size); - strm_.next_in = reinterpret_cast(const_cast(data)); - - char outbuf[8192]; - do { - strm_.avail_out = sizeof(outbuf); - strm_.next_out = reinterpret_cast(outbuf); - - int ret = deflate(&strm_, flush_mode); - if (ret == Z_STREAM_ERROR) { - throw std::runtime_error("Gzip compression failed"); - } - - size_t have = sizeof(outbuf) - strm_.avail_out; - result.append(outbuf, have); - } while (strm_.avail_out == 0); - - return result; - } - - std::string finish() { return compress(nullptr, 0, Z_FINISH); } - -private: - z_stream strm_{}; -}; - -[[nodiscard]] AuthHeaders authHeadersFromRequest(const drogon::HttpRequestPtr& req) -{ - AuthHeaders headers; - for (auto const& [k, v] : req->headers()) { - headers.emplace(k, v); - } - return headers; -} - -[[nodiscard]] bool containsGzip(std::string_view acceptEncoding) -{ - return !acceptEncoding.empty() && acceptEncoding.find("gzip") != std::string_view::npos; -} - -} // namespace - -struct HttpService::Impl -{ - HttpService& self_; - HttpServiceConfig config_; - mutable std::atomic binaryRequestCounter_{0}; - mutable std::atomic jsonRequestCounter_{0}; - - explicit Impl(HttpService& self, const HttpServiceConfig& config) : self_(self), config_(config) {} - - enum class ResponseType { Binary, Json }; - - void tryMemoryTrim(ResponseType responseType) const - { - uint64_t interval = - (responseType == ResponseType::Binary) ? config_.memoryTrimIntervalBinary : config_.memoryTrimIntervalJson; - - if (interval == 0) { - return; - } - - auto& counter = (responseType == ResponseType::Binary) ? binaryRequestCounter_ : jsonRequestCounter_; - auto count = counter.fetch_add(1, std::memory_order_relaxed); - if ((count % interval) != 0) { - return; - } - -#ifdef __linux__ -#ifndef NDEBUG - const char* typeStr = (responseType == ResponseType::Binary) ? "binary" : "JSON"; - log().debug("Trimming memory after {} {} requests (interval: {})", count, typeStr, interval); -#endif - malloc_trim(0); -#endif - } - - struct TilesStreamState : std::enable_shared_from_this - { - static constexpr auto binaryMimeType = "application/binary"; - static constexpr auto jsonlMimeType = "application/jsonl"; - static constexpr auto anyMimeType = "*/*"; - - explicit TilesStreamState(Impl const& impl, trantor::EventLoop* loop) : impl_(impl), loop_(loop) - { - static std::atomic_uint64_t nextRequestId; - requestId_ = nextRequestId++; - writer_ = std::make_unique( - [this](auto&& msg, auto&& /*msgType*/) { appendOutgoingUnlocked(msg); }, stringOffsets_); - } - - void attachStream(drogon::ResponseStreamPtr stream) - { - { - std::lock_guard lock(mutex_); - if (aborted_ || responseEnded_) { - if (stream) - stream->close(); - return; - } - stream_ = std::move(stream); - } - scheduleDrain(); - } - - void parseRequestFromJson(nlohmann::json const& requestJson) - { - std::string mapId = requestJson["mapId"]; - std::string layerId = requestJson["layerId"]; - std::vector tileIds; - tileIds.reserve(requestJson["tileIds"].size()); - for (auto const& tid : requestJson["tileIds"].get>()) { - tileIds.emplace_back(tid); - } - requests_.push_back(std::make_shared(mapId, layerId, std::move(tileIds))); - } - - [[nodiscard]] bool setResponseTypeFromAccept(std::string_view acceptHeader, std::string& error) - { - responseType_ = std::string(acceptHeader); - if (responseType_.empty()) - responseType_ = anyMimeType; - if (responseType_ == anyMimeType) - responseType_ = binaryMimeType; - - if (responseType_ == binaryMimeType) { - trimResponseType_ = ResponseType::Binary; - return true; - } - if (responseType_ == jsonlMimeType) { - trimResponseType_ = ResponseType::Json; - return true; - } - - error = "Unknown Accept header value: " + responseType_; - return false; - } - - void enableGzip() { compressor_ = std::make_unique(); } - - void onAborted() - { - if (aborted_.exchange(true)) - return; - for (auto const& req : requests_) { - if (!req->isDone()) { - impl_.self_.abort(req); - } - } - drogon::ResponseStreamPtr stream; - { - std::lock_guard lock(mutex_); - if (responseEnded_.exchange(true)) - return; - stream = std::move(stream_); - } - if (stream) - stream->close(); - } - - void addResult(TileLayer::Ptr const& result) - { - { - std::lock_guard lock(mutex_); - if (aborted_) - return; - - log().debug("Response ready: {}", MapTileKey(*result).toString()); - if (responseType_ == binaryMimeType) { - writer_->write(result); - } else { - auto dumped = result->toJson().dump( - -1, ' ', false, nlohmann::json::error_handler_t::ignore); - appendOutgoingUnlocked(dumped); - appendOutgoingUnlocked("\n"); - } - } - scheduleDrain(); - } - - void onRequestDone() - { - { - std::lock_guard lock(mutex_); - if (aborted_) - return; - - bool allDoneNow = std::all_of( - requests_.begin(), requests_.end(), [](auto const& r) { return r->isDone(); }); - - if (allDoneNow && !allDone_) { - allDone_ = true; - if (responseType_ == binaryMimeType && !endOfStreamSent_) { - writer_->sendEndOfStream(); - endOfStreamSent_ = true; - } - } - } - scheduleDrain(); - } - - void scheduleDrain() - { - if (aborted_ || responseEnded_) - return; - if (drainScheduled_.exchange(true)) - return; - - auto weak = weak_from_this(); - loop_->queueInLoop([weak = std::move(weak)]() mutable { - if (auto self = weak.lock()) { - self->drainOnLoop(); - } - }); - } - - void drainOnLoop() - { - drainScheduled_ = false; - if (aborted_ || responseEnded_) - return; - - constexpr size_t maxChunk = 64 * 1024; - - for (;;) { - std::string chunk; - bool done = false; - bool needAbort = false; - bool scheduleAgain = false; - drogon::ResponseStreamPtr streamToClose; - { - std::lock_guard lock(mutex_); - if (!stream_) - return; - - if (!pending_.empty()) { - size_t n = std::min(pending_.size(), maxChunk); - chunk.assign(pending_.data(), n); - pending_.erase(0, n); - } else { - if (allDone_ && compressor_ && !compressionFinished_) { - pending_.append(compressor_->finish()); - compressionFinished_ = true; - continue; - } - done = allDone_; - } - - if (!chunk.empty()) { - if (!stream_->send(chunk)) { - needAbort = true; - } else if (!pending_.empty() || allDone_) { - // Keep draining until we sent everything and closed the stream. - scheduleAgain = true; - } - } else if (done) { - responseEnded_ = true; - streamToClose = std::move(stream_); - } - } - - if (needAbort) { - onAborted(); - return; - } - - if (done) { - if (streamToClose) - streamToClose->close(); - impl_.tryMemoryTrim(trimResponseType_); - return; - } - if (scheduleAgain) - scheduleDrain(); - return; - } - } - - void appendOutgoingUnlocked(std::string_view bytes) - { - if (bytes.empty()) - return; - - if (compressor_) { - pending_.append(compressor_->compress(bytes.data(), bytes.size())); - } else { - pending_.append(bytes); - } - } - - Impl const& impl_; - trantor::EventLoop* loop_; - - std::mutex mutex_; - uint64_t requestId_ = 0; - - std::string responseType_; - ResponseType trimResponseType_ = ResponseType::Binary; - - std::string pending_; - drogon::ResponseStreamPtr stream_; - std::unique_ptr writer_; - std::vector requests_; - TileLayerStream::StringPoolOffsetMap stringOffsets_; - - std::unique_ptr compressor_; - bool compressionFinished_ = false; - bool endOfStreamSent_ = false; - bool allDone_ = false; - - std::atomic_bool aborted_{false}; - std::atomic_bool drainScheduled_{false}; - std::atomic_bool responseEnded_{false}; - }; - - mutable std::mutex clientRequestMapMutex_; - mutable std::unordered_map> requestStatePerClientId_; - - void abortRequestsForClientId( - std::string const& clientId, - std::shared_ptr newState = nullptr) const - { - std::unique_lock clientRequestMapAccess(clientRequestMapMutex_); - auto clientRequestIt = requestStatePerClientId_.find(clientId); - if (clientRequestIt != requestStatePerClientId_.end()) { - bool anySoftAbort = false; - for (auto const& req : clientRequestIt->second->requests_) { - if (!req->isDone()) { - self_.abort(req); - anySoftAbort = true; - } - } - if (anySoftAbort) - log().warn("Soft-aborting tiles request {}", clientRequestIt->second->requestId_); - requestStatePerClientId_.erase(clientRequestIt); - } - if (newState) { - requestStatePerClientId_.emplace(clientId, std::move(newState)); - } - } - - void handleTilesRequest( - const drogon::HttpRequestPtr& req, - std::function&& callback) const - { - auto state = std::make_shared(*this, drogon::app().getLoop()); - - const std::string accept = req->getHeader("accept"); - const std::string acceptEncoding = req->getHeader("accept-encoding"); - auto clientHeaders = authHeadersFromRequest(req); - - nlohmann::json j; - try { - j = nlohmann::json::parse(std::string(req->body())); - } - catch (const std::exception& e) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k400BadRequest); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::string("Invalid JSON: ") + e.what()); - callback(resp); - return; - } - - auto requestsIt = j.find("requests"); - if (requestsIt == j.end() || !requestsIt->is_array()) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k400BadRequest); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("Missing or invalid 'requests' array"); - callback(resp); - return; - } - - log().info("Processing tiles request {}", state->requestId_); - for (auto& requestJson : *requestsIt) { - state->parseRequestFromJson(requestJson); - } - - if (j.contains("stringPoolOffsets")) { - for (auto& item : j["stringPoolOffsets"].items()) { - state->stringOffsets_[item.key()] = item.value().get(); - } - } - - std::string acceptError; - if (!state->setResponseTypeFromAccept(accept, acceptError)) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k400BadRequest); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::move(acceptError)); - callback(resp); - return; - } - - const bool gzip = containsGzip(acceptEncoding); - if (gzip) { - state->enableGzip(); - } - - for (auto& request : state->requests_) { - request->onFeatureLayer([state](auto&& layer) { state->addResult(layer); }); - request->onSourceDataLayer([state](auto&& layer) { state->addResult(layer); }); - request->onDone_ = [state](RequestStatus) { state->onRequestDone(); }; - } - - const auto canProcess = self_.request(state->requests_, clientHeaders); - if (!canProcess) { - std::vector> requestStatuses{}; - bool anyUnauthorized = false; - for (auto const& r : state->requests_) { - auto status = r->getStatus(); - requestStatuses.emplace_back(static_cast>(status)); - anyUnauthorized |= (status == RequestStatus::Unauthorized); - } - - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(anyUnauthorized ? drogon::k403Forbidden : drogon::k400BadRequest); - resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); - resp->setBody(nlohmann::json::object({{"status", requestStatuses}}).dump()); - callback(resp); - return; - } - - if (j.contains("clientId")) { - abortRequestsForClientId(j["clientId"].get(), state); - } - - auto resp = drogon::HttpResponse::newAsyncStreamResponse( - [state](drogon::ResponseStreamPtr stream) { state->attachStream(std::move(stream)); }, - true); - resp->setStatusCode(drogon::k200OK); - resp->setContentTypeString(state->responseType_); - if (gzip) { - resp->addHeader("Content-Encoding", "gzip"); - } - callback(resp); - } - - void handleAbortRequest( - const drogon::HttpRequestPtr& req, - std::function&& callback) const - { - try { - auto j = nlohmann::json::parse(std::string(req->body())); - if (j.contains("clientId")) { - abortRequestsForClientId(j["clientId"].get()); - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k200OK); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("OK"); - callback(resp); - return; - } - - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k400BadRequest); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("Missing clientId"); - callback(resp); - } - catch (const std::exception& e) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k400BadRequest); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::string("Invalid JSON: ") + e.what()); - callback(resp); - } - } - - void handleSourcesRequest( - const drogon::HttpRequestPtr& req, - std::function&& callback) const - { - auto sourcesInfo = nlohmann::json::array(); - for (auto& source : self_.info(authHeadersFromRequest(req))) { - sourcesInfo.push_back(source.toJson()); - } - - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k200OK); - resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); - resp->setBody(sourcesInfo.dump()); - callback(resp); - } - - void handleStatusRequest( - const drogon::HttpRequestPtr&, - std::function&& callback) const - { - auto serviceStats = self_.getStatistics(); - auto cacheStats = self_.cache()->getStatistics(); - - std::ostringstream oss; - oss << ""; - oss << "

Status Information

"; - oss << "

Service Statistics

"; - oss << "
" << serviceStats.dump(4) << "
"; - oss << "

Cache Statistics

"; - oss << "
" << cacheStats.dump(4) << "
"; - oss << ""; - - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k200OK); - resp->setContentTypeCode(drogon::CT_TEXT_HTML); - resp->setBody(oss.str()); - callback(resp); - } - - void handleLocateRequest( - const drogon::HttpRequestPtr& req, - std::function&& callback) const - { - try { - nlohmann::json j = nlohmann::json::parse(std::string(req->body())); - auto requestsJson = j["requests"]; - auto allResponsesJson = nlohmann::json::array(); - - for (auto const& locateReqJson : requestsJson) { - LocateRequest locateReq{locateReqJson}; - auto responsesJson = nlohmann::json::array(); - for (auto const& resp : self_.locate(locateReq)) - responsesJson.emplace_back(resp.serialize()); - allResponsesJson.emplace_back(responsesJson); - } - - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k200OK); - resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); - resp->setBody(nlohmann::json::object({{"responses", allResponsesJson}}).dump()); - callback(resp); - } - catch (const std::exception& e) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k400BadRequest); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::string("Invalid JSON: ") + e.what()); - callback(resp); - } - } - - static drogon::HttpResponsePtr openConfigFile(std::ifstream& configFile) - { - auto configFilePath = DataSourceConfigService::get().getConfigFilePath(); - if (!configFilePath.has_value()) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k404NotFound); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("The config file path is not set. Check the server configuration."); - return resp; - } - - std::filesystem::path path = *configFilePath; - if (!std::filesystem::exists(path)) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k404NotFound); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("The server does not have a config file."); - return resp; - } - - configFile.open(*configFilePath); - if (!configFile) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k500InternalServerError); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("Failed to open config file."); - return resp; - } - - return nullptr; - } - - static void handleGetConfigRequest( - const drogon::HttpRequestPtr&, - std::function&& callback) - { - if (!isGetConfigEndpointEnabled()) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k403Forbidden); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("The GET /config endpoint is disabled by the server administrator."); - callback(resp); - return; - } - - std::ifstream configFile; - if (auto errorResp = openConfigFile(configFile)) { - callback(errorResp); - return; - } - - nlohmann::json jsonSchema = DataSourceConfigService::get().getDataSourceConfigSchema(); - - try { - YAML::Node configYaml = YAML::Load(configFile); - nlohmann::json jsonConfig; - std::unordered_map maskedSecretMap; - for (const auto& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { - if (auto configYamlEntry = configYaml[key]) - jsonConfig[key] = yamlToJson(configYaml[key], true, &maskedSecretMap); - } - - nlohmann::json combinedJson; - combinedJson["schema"] = jsonSchema; - combinedJson["model"] = jsonConfig; - combinedJson["readOnly"] = !isPostConfigEndpointEnabled(); - - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k200OK); - resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); - resp->setBody(combinedJson.dump(2)); - callback(resp); - } - catch (const std::exception& e) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k500InternalServerError); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::string("Error processing config file: ") + e.what()); - callback(resp); - } - } - - void handlePostConfigRequest( - const drogon::HttpRequestPtr& req, - std::function&& callback) const - { - if (!isPostConfigEndpointEnabled()) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k403Forbidden); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("The POST /config endpoint is not enabled by the server administrator."); - callback(resp); - return; - } - - struct ConfigUpdateState : std::enable_shared_from_this - { - trantor::EventLoop* loop = nullptr; - std::atomic_bool done{false}; - std::atomic_bool wroteConfig{false}; - std::unique_ptr subscription; - std::function callback; - }; - - std::ifstream configFile; - if (auto errorResp = openConfigFile(configFile)) { - callback(errorResp); - return; - } - - nlohmann::json jsonConfig; - try { - jsonConfig = nlohmann::json::parse(std::string(req->body())); - } - catch (const nlohmann::json::parse_error& e) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k400BadRequest); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::string("Invalid JSON format: ") + e.what()); - callback(resp); - return; - } - - try { - DataSourceConfigService::get().validateDataSourceConfig(jsonConfig); - } - catch (const std::exception& e) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k500InternalServerError); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::string("Validation failed: ") + e.what()); - callback(resp); - return; - } - - auto yamlConfig = YAML::Load(configFile); - std::unordered_map maskedSecrets; - yamlToJson(yamlConfig, true, &maskedSecrets); - - for (auto const& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { - if (jsonConfig.contains(key)) - yamlConfig[key] = jsonToYaml(jsonConfig[key], maskedSecrets); - } - - auto state = std::make_shared(); - state->loop = drogon::app().getLoop(); - state->callback = std::move(callback); - - // Subscribe before writing; ignore any callbacks that happen before we write. - state->subscription = DataSourceConfigService::get().subscribe( - [state](std::vector const&) mutable { - if (!state->wroteConfig) { - return; - } - if (state->done.exchange(true)) - return; - state->loop->queueInLoop([state]() mutable { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k200OK); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("Configuration updated and applied successfully."); - state->callback(resp); - state->subscription.reset(); - }); - }, - [state](std::string const& error) mutable { - if (!state->wroteConfig) { - return; - } - if (state->done.exchange(true)) - return; - state->loop->queueInLoop([state, error]() mutable { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k500InternalServerError); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::string("Error applying the configuration: ") + error); - state->callback(resp); - state->subscription.reset(); - }); - }); - - configFile.close(); - log().trace("Writing new config."); - state->wroteConfig = true; - if (auto configFilePath = DataSourceConfigService::get().getConfigFilePath()) { - std::ofstream newConfigFile(*configFilePath); - newConfigFile << yamlConfig; - newConfigFile.close(); - } - - // Timeout fail-safe (rare endpoint; ok to spawn a thread). - std::thread([weak = state->weak_from_this()]() { - std::this_thread::sleep_for(std::chrono::seconds(60)); - if (auto state = weak.lock()) { - if (state->done.exchange(true)) - return; - state->loop->queueInLoop([state]() mutable { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k500InternalServerError); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("Timeout while waiting for config to update."); - state->callback(resp); - state->subscription.reset(); - }); - } - }).detach(); - } -}; - HttpService::HttpService(Cache::Ptr cache, const HttpServiceConfig& config) : Service(std::move(cache), config.watchConfig, config.defaultTtl), impl_(std::make_unique(*this, config)) { @@ -828,6 +20,8 @@ HttpService::~HttpService() = default; void HttpService::setup(drogon::HttpAppFramework& app) { + detail::registerTilesWebSocketController(app, *this); + app.registerHandler( "/tiles", [this](const drogon::HttpRequestPtr& req, std::function&& callback) { @@ -835,13 +29,6 @@ void HttpService::setup(drogon::HttpAppFramework& app) }, {drogon::Post}); - app.registerHandler( - "/abort", - [this](const drogon::HttpRequestPtr& req, std::function&& callback) { - impl_->handleAbortRequest(req, std::move(callback)); - }, - {drogon::Post}); - app.registerHandler( "/sources", [this](const drogon::HttpRequestPtr& req, std::function&& callback) { diff --git a/libs/http-service/src/locate-handler.cpp b/libs/http-service/src/locate-handler.cpp new file mode 100644 index 00000000..81ad9ea5 --- /dev/null +++ b/libs/http-service/src/locate-handler.cpp @@ -0,0 +1,43 @@ +#include "http-service-impl.h" + +#include + +#include "nlohmann/json.hpp" + +namespace mapget +{ + +void HttpService::Impl::handleLocateRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const +{ + try { + nlohmann::json j = nlohmann::json::parse(std::string(req->body())); + auto requestsJson = j["requests"]; + auto allResponsesJson = nlohmann::json::array(); + + for (auto const& locateReqJson : requestsJson) { + LocateRequest locateReq{locateReqJson}; + auto responsesJson = nlohmann::json::array(); + for (auto const& resp : self_.locate(locateReq)) + responsesJson.emplace_back(resp.serialize()); + allResponsesJson.emplace_back(responsesJson); + } + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(nlohmann::json::object({{"responses", allResponsesJson}}).dump()); + callback(resp); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON: ") + e.what()); + callback(resp); + } +} + +} // namespace mapget + diff --git a/libs/http-service/src/sources-handler.cpp b/libs/http-service/src/sources-handler.cpp new file mode 100644 index 00000000..d085ac3f --- /dev/null +++ b/libs/http-service/src/sources-handler.cpp @@ -0,0 +1,27 @@ +#include "http-service-impl.h" + +#include + +#include "nlohmann/json.hpp" + +namespace mapget +{ + +void HttpService::Impl::handleSourcesRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const +{ + auto sourcesInfo = nlohmann::json::array(); + for (auto& source : self_.info(detail::authHeadersFromRequest(req))) { + sourcesInfo.push_back(source.toJson()); + } + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(sourcesInfo.dump()); + callback(resp); +} + +} // namespace mapget + diff --git a/libs/http-service/src/status-handler.cpp b/libs/http-service/src/status-handler.cpp new file mode 100644 index 00000000..b0a91216 --- /dev/null +++ b/libs/http-service/src/status-handler.cpp @@ -0,0 +1,34 @@ +#include "http-service-impl.h" + +#include + +#include + +namespace mapget +{ + +void HttpService::Impl::handleStatusRequest( + const drogon::HttpRequestPtr& /*req*/, + std::function&& callback) const +{ + auto serviceStats = self_.getStatistics(); + auto cacheStats = self_.cache()->getStatistics(); + + std::ostringstream oss; + oss << ""; + oss << "

Status Information

"; + oss << "

Service Statistics

"; + oss << "
" << serviceStats.dump(4) << "
"; + oss << "

Cache Statistics

"; + oss << "
" << cacheStats.dump(4) << "
"; + oss << ""; + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_TEXT_HTML); + resp->setBody(oss.str()); + callback(resp); +} + +} // namespace mapget + diff --git a/libs/http-service/src/tiles-http-handler.cpp b/libs/http-service/src/tiles-http-handler.cpp new file mode 100644 index 00000000..3edaa1ad --- /dev/null +++ b/libs/http-service/src/tiles-http-handler.cpp @@ -0,0 +1,420 @@ +#include "http-service-impl.h" + +#include "mapget/log.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nlohmann/json.hpp" + +#include + +namespace mapget +{ +namespace +{ + +class GzipCompressor +{ +public: + GzipCompressor() + { + strm_.zalloc = Z_NULL; + strm_.zfree = Z_NULL; + strm_.opaque = Z_NULL; + // 16+MAX_WBITS enables gzip format (not just deflate) + int ret = deflateInit2( + &strm_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 16 + MAX_WBITS, 8, Z_DEFAULT_STRATEGY); + if (ret != Z_OK) { + throw std::runtime_error("Failed to initialize gzip compressor"); + } + } + + ~GzipCompressor() { deflateEnd(&strm_); } + + GzipCompressor(GzipCompressor const&) = delete; + GzipCompressor(GzipCompressor&&) = delete; + + std::string compress(const char* data, size_t size, int flush_mode = Z_NO_FLUSH) + { + std::string result; + if (size == 0 && flush_mode == Z_NO_FLUSH) { + return result; + } + + strm_.avail_in = static_cast(size); + strm_.next_in = reinterpret_cast(const_cast(data)); + + char outbuf[8192]; + do { + strm_.avail_out = sizeof(outbuf); + strm_.next_out = reinterpret_cast(outbuf); + + int ret = deflate(&strm_, flush_mode); + if (ret == Z_STREAM_ERROR) { + throw std::runtime_error("Gzip compression failed"); + } + + size_t have = sizeof(outbuf) - strm_.avail_out; + result.append(outbuf, have); + } while (strm_.avail_out == 0); + + return result; + } + + std::string finish() { return compress(nullptr, 0, Z_FINISH); } + +private: + z_stream strm_{}; +}; + +[[nodiscard]] bool containsGzip(std::string_view acceptEncoding) +{ + return !acceptEncoding.empty() && acceptEncoding.find("gzip") != std::string_view::npos; +} + +} // namespace + +struct HttpService::Impl::TilesStreamState : std::enable_shared_from_this +{ + static constexpr auto binaryMimeType = "application/binary"; + static constexpr auto jsonlMimeType = "application/jsonl"; + static constexpr auto anyMimeType = "*/*"; + + explicit TilesStreamState(Impl const& impl, trantor::EventLoop* loop) : impl_(impl), loop_(loop) + { + static std::atomic_uint64_t nextRequestId; + requestId_ = nextRequestId++; + writer_ = std::make_unique( + [this](auto&& msg, auto&& /*msgType*/) { appendOutgoingUnlocked(msg); }, stringOffsets_); + } + + void attachStream(drogon::ResponseStreamPtr stream) + { + { + std::lock_guard lock(mutex_); + if (aborted_ || responseEnded_) { + if (stream) + stream->close(); + return; + } + stream_ = std::move(stream); + } + scheduleDrain(); + } + + void parseRequestFromJson(nlohmann::json const& requestJson) + { + std::string mapId = requestJson["mapId"]; + std::string layerId = requestJson["layerId"]; + std::vector tileIds; + tileIds.reserve(requestJson["tileIds"].size()); + for (auto const& tid : requestJson["tileIds"].get>()) { + tileIds.emplace_back(tid); + } + requests_.push_back(std::make_shared(mapId, layerId, std::move(tileIds))); + } + + [[nodiscard]] bool setResponseTypeFromAccept(std::string_view acceptHeader, std::string& error) + { + responseType_ = std::string(acceptHeader); + if (responseType_.empty()) + responseType_ = anyMimeType; + if (responseType_ == anyMimeType) + responseType_ = binaryMimeType; + + if (responseType_ == binaryMimeType) { + trimResponseType_ = HttpService::Impl::ResponseType::Binary; + return true; + } + if (responseType_ == jsonlMimeType) { + trimResponseType_ = HttpService::Impl::ResponseType::Json; + return true; + } + + error = "Unknown Accept header value: " + responseType_; + return false; + } + + void enableGzip() { compressor_ = std::make_unique(); } + + void onAborted() + { + if (aborted_.exchange(true)) + return; + for (auto const& req : requests_) { + if (!req->isDone()) { + impl_.self_.abort(req); + } + } + drogon::ResponseStreamPtr stream; + { + std::lock_guard lock(mutex_); + if (responseEnded_.exchange(true)) + return; + stream = std::move(stream_); + } + if (stream) + stream->close(); + } + + void addResult(TileLayer::Ptr const& result) + { + { + std::lock_guard lock(mutex_); + if (aborted_) + return; + + log().debug("Response ready: {}", MapTileKey(*result).toString()); + if (responseType_ == binaryMimeType) { + writer_->write(result); + } else { + auto dumped = result->toJson().dump(-1, ' ', false, nlohmann::json::error_handler_t::ignore); + appendOutgoingUnlocked(dumped); + appendOutgoingUnlocked("\n"); + } + } + scheduleDrain(); + } + + void onRequestDone() + { + { + std::lock_guard lock(mutex_); + if (aborted_) + return; + + bool allDoneNow = + std::all_of(requests_.begin(), requests_.end(), [](auto const& r) { return r->isDone(); }); + + if (allDoneNow && !allDone_) { + allDone_ = true; + if (responseType_ == binaryMimeType && !endOfStreamSent_) { + writer_->sendEndOfStream(); + endOfStreamSent_ = true; + } + } + } + scheduleDrain(); + } + + void scheduleDrain() + { + if (aborted_ || responseEnded_) + return; + if (drainScheduled_.exchange(true)) + return; + + auto weak = weak_from_this(); + loop_->queueInLoop([weak = std::move(weak)]() mutable { + if (auto self = weak.lock()) { + self->drainOnLoop(); + } + }); + } + + void drainOnLoop() + { + drainScheduled_ = false; + if (aborted_ || responseEnded_) + return; + + constexpr size_t maxChunk = 64 * 1024; + + for (;;) { + std::string chunk; + bool done = false; + bool needAbort = false; + bool scheduleAgain = false; + drogon::ResponseStreamPtr streamToClose; + { + std::lock_guard lock(mutex_); + if (!stream_) + return; + + if (!pending_.empty()) { + size_t n = std::min(pending_.size(), maxChunk); + chunk.assign(pending_.data(), n); + pending_.erase(0, n); + } else { + if (allDone_ && compressor_ && !compressionFinished_) { + pending_.append(compressor_->finish()); + compressionFinished_ = true; + continue; + } + done = allDone_; + } + + if (!chunk.empty()) { + if (!stream_->send(chunk)) { + needAbort = true; + } else if (!pending_.empty() || allDone_) { + scheduleAgain = true; + } + } else if (done) { + responseEnded_ = true; + streamToClose = std::move(stream_); + } + } + + if (needAbort) { + onAborted(); + return; + } + + if (done) { + if (streamToClose) + streamToClose->close(); + impl_.tryMemoryTrim(trimResponseType_); + return; + } + if (scheduleAgain) + scheduleDrain(); + return; + } + } + + void appendOutgoingUnlocked(std::string_view bytes) + { + if (bytes.empty()) + return; + + if (compressor_) { + pending_.append(compressor_->compress(bytes.data(), bytes.size())); + } else { + pending_.append(bytes); + } + } + + Impl const& impl_; + trantor::EventLoop* loop_; + + std::mutex mutex_; + uint64_t requestId_ = 0; + + std::string responseType_; + HttpService::Impl::ResponseType trimResponseType_ = HttpService::Impl::ResponseType::Binary; + + std::string pending_; + drogon::ResponseStreamPtr stream_; + std::unique_ptr writer_; + std::vector requests_; + TileLayerStream::StringPoolOffsetMap stringOffsets_; + + std::unique_ptr compressor_; + bool compressionFinished_ = false; + bool endOfStreamSent_ = false; + bool allDone_ = false; + + std::atomic_bool aborted_{false}; + std::atomic_bool drainScheduled_{false}; + std::atomic_bool responseEnded_{false}; +}; + +void HttpService::Impl::handleTilesRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const +{ + auto state = std::make_shared(*this, drogon::app().getLoop()); + + const std::string accept = req->getHeader("accept"); + const std::string acceptEncoding = req->getHeader("accept-encoding"); + auto clientHeaders = detail::authHeadersFromRequest(req); + + nlohmann::json j; + try { + j = nlohmann::json::parse(std::string(req->body())); + } + catch (const std::exception& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid JSON: ") + e.what()); + callback(resp); + return; + } + + auto requestsIt = j.find("requests"); + if (requestsIt == j.end() || !requestsIt->is_array()) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody("Missing or invalid 'requests' array"); + callback(resp); + return; + } + + log().info("Processing tiles request {}", state->requestId_); + for (auto& requestJson : *requestsIt) { + state->parseRequestFromJson(requestJson); + } + + if (j.contains("stringPoolOffsets")) { + for (auto& item : j["stringPoolOffsets"].items()) { + state->stringOffsets_[item.key()] = item.value().get(); + } + } + + std::string acceptError; + if (!state->setResponseTypeFromAccept(accept, acceptError)) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::move(acceptError)); + callback(resp); + return; + } + + const bool gzip = containsGzip(acceptEncoding); + if (gzip) { + state->enableGzip(); + } + + for (auto& request : state->requests_) { + request->onFeatureLayer([state](auto&& layer) { state->addResult(layer); }); + request->onSourceDataLayer([state](auto&& layer) { state->addResult(layer); }); + request->onDone_ = [state](RequestStatus) { state->onRequestDone(); }; + } + + const auto canProcess = self_.request(state->requests_, clientHeaders); + if (!canProcess) { + std::vector> requestStatuses{}; + bool anyUnauthorized = false; + for (auto const& r : state->requests_) { + auto status = r->getStatus(); + requestStatuses.emplace_back(static_cast>(status)); + anyUnauthorized |= (status == RequestStatus::Unauthorized); + } + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(anyUnauthorized ? drogon::k403Forbidden : drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(nlohmann::json::object({{"status", requestStatuses}}).dump()); + callback(resp); + return; + } + + auto resp = drogon::HttpResponse::newAsyncStreamResponse( + [state](drogon::ResponseStreamPtr stream) { state->attachStream(std::move(stream)); }, + true); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeString(state->responseType_); + if (gzip) { + resp->addHeader("Content-Encoding", "gzip"); + } + callback(resp); +} + +} // namespace mapget diff --git a/libs/http-service/src/tiles-ws-controller.cpp b/libs/http-service/src/tiles-ws-controller.cpp new file mode 100644 index 00000000..82fefac9 --- /dev/null +++ b/libs/http-service/src/tiles-ws-controller.cpp @@ -0,0 +1,558 @@ +#include "tiles-ws-controller.h" + +#include "mapget/http-service/http-service.h" + +#include "mapget/log.h" +#include "mapget/model/stream.h" + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "nlohmann/json.hpp" + +namespace mapget::detail +{ +namespace +{ + +[[nodiscard]] AuthHeaders authHeadersFromRequest(const drogon::HttpRequestPtr& req) +{ + AuthHeaders headers; + for (auto const& [k, v] : req->headers()) { + headers.emplace(k, v); + } + return headers; +} + +[[nodiscard]] std::string_view requestStatusToString(RequestStatus s) +{ + switch (s) { + case RequestStatus::Open: + return "Open"; + case RequestStatus::Success: + return "Success"; + case RequestStatus::NoDataSource: + return "NoDataSource"; + case RequestStatus::Unauthorized: + return "Unauthorized"; + case RequestStatus::Aborted: + return "Aborted"; + } + return "Unknown"; +} + +[[nodiscard]] std::string encodeStreamMessage(TileLayerStream::MessageType type, std::string_view payload) +{ + std::ostringstream headerStream; + bitsery::Serializer s(headerStream); + s.object(TileLayerStream::CurrentProtocolVersion); + s.value1b(type); + s.value4b(static_cast(payload.size())); + + auto message = headerStream.str(); + message.append(payload); + return message; +} + +struct WsConnectionState +{ + AuthHeaders authHeaders; + TileLayerStream::StringPoolOffsetMap stringPoolOffsets; + std::shared_ptr session; +}; + +class TilesWsSession : public std::enable_shared_from_this +{ +public: + TilesWsSession( + HttpService& service, + std::weak_ptr conn, + std::weak_ptr connState, + AuthHeaders authHeaders, + TileLayerStream::StringPoolOffsetMap initialOffsets) + : service_(service), + loop_(drogon::app().getLoop()), + conn_(std::move(conn)), + connState_(std::move(connState)), + authHeaders_(std::move(authHeaders)), + offsets_(std::move(initialOffsets)), + writer_( + std::make_unique( + [this](std::string msg, TileLayerStream::MessageType type) { onWriterMessage(std::move(msg), type); }, + offsets_)) + { + } + + ~TilesWsSession() + { + // Best-effort cleanup: abort any in-flight requests if the session is destroyed. + cancelNoStatus(); + } + + TilesWsSession(TilesWsSession const&) = delete; + TilesWsSession& operator=(TilesWsSession const&) = delete; + + void start(const nlohmann::json& j) + { + auto requestsIt = j.find("requests"); + if (requestsIt == j.end() || !requestsIt->is_array()) { + queueStatusMessage("Missing or invalid 'requests' array"); + scheduleDrain(); + return; + } + + try { + requests_.clear(); + requests_.reserve(requestsIt->size()); + requestStatuses_.clear(); + requestStatuses_.reserve(requestsIt->size()); + + for (auto const& requestJson : *requestsIt) { + const std::string mapId = requestJson.at("mapId").get(); + const std::string layerId = requestJson.at("layerId").get(); + const auto& tileIdsJson = requestJson.at("tileIds"); + if (!tileIdsJson.is_array()) { + throw std::runtime_error("tileIds must be an array"); + } + + std::vector tileIds; + tileIds.reserve(tileIdsJson.size()); + for (auto const& tid : tileIdsJson) { + tileIds.emplace_back(tid.get()); + } + + requests_.push_back(std::make_shared(mapId, layerId, std::move(tileIds))); + requestStatuses_.push_back(RequestStatus::Open); + } + } + catch (const std::exception& e) { + queueStatusMessage(fmt::format("Invalid request JSON: {}", e.what())); + scheduleDrain(); + return; + } + + // Hook request callbacks before calling service_.request so early + // failures (NoDataSource/Unauthorized) still produce status updates. + const auto weak = weak_from_this(); + for (size_t i = 0; i < requests_.size(); ++i) { + auto& req = requests_[i]; + req->onFeatureLayer([weak](auto&& layer) { + if (auto self = weak.lock()) { + self->onTileLayer(std::move(layer)); + } + }); + req->onSourceDataLayer([weak](auto&& layer) { + if (auto self = weak.lock()) { + self->onTileLayer(std::move(layer)); + } + }); + req->onDone_ = [weak, i](RequestStatus status) { + if (auto self = weak.lock()) { + self->onRequestDone(i, status); + } + }; + } + + // Start processing (may synchronously set request statuses). + (void)service_.request(requests_, authHeaders_); + + { + std::lock_guard lock(mutex_); + statusEmissionEnabled_ = true; + } + queueStatusMessage({}); + scheduleDrain(); + } + + void cancel(std::string reason) + { + cancelled_ = true; + + // Stop sending any queued tile frames from this session. + { + std::lock_guard lock(mutex_); + outgoing_.clear(); + } + + // Abort in-flight requests (best-effort). + for (auto const& r : requests_) { + if (!r || r->isDone()) + continue; + service_.abort(r); + } + + // Refresh locally cached statuses after aborting. + { + std::lock_guard lock(mutex_); + for (size_t i = 0; i < requests_.size() && i < requestStatuses_.size(); ++i) { + if (requests_[i]) { + requestStatuses_[i] = requests_[i]->getStatus(); + } + } + } + + queueStatusMessage(std::move(reason)); + scheduleDrain(); + } + +private: + struct OutgoingFrame + { + std::string bytes; + std::optional> stringPoolCommit; + }; + + struct WriterMessage + { + std::string bytes; + TileLayerStream::MessageType type{TileLayerStream::MessageType::None}; + }; + + void cancelNoStatus() + { + if (cancelled_.exchange(true)) + return; + + // Ensure we stop emitting any further frames. + { + std::lock_guard lock(mutex_); + outgoing_.clear(); + } + + for (auto const& r : requests_) { + if (!r || r->isDone()) + continue; + service_.abort(r); + } + } + + void onWriterMessage(std::string msg, TileLayerStream::MessageType type) + { + // Writer messages are only generated from within onTileLayer under mutex_. + if (!currentWriteBatch_) { + raise("TilesWsSession writer callback used out-of-band"); + } + currentWriteBatch_->push_back(WriterMessage{std::move(msg), type}); + } + + void onTileLayer(TileLayer::Ptr layer) + { + if (cancelled_) + return; + if (!layer) + return; + + std::vector batch; + std::optional> stringPoolCommit; + + { + std::lock_guard lock(mutex_); + if (cancelled_) + return; + + currentWriteBatch_ = &batch; + writer_->write(layer); + currentWriteBatch_ = nullptr; + + // If a StringPool message was generated, the writer updates offsets_ + // to the new highest string ID for this node after emitting it. + const auto nodeId = layer->nodeId(); + const auto it = offsets_.find(nodeId); + if (it != offsets_.end()) { + const auto newOffset = it->second; + for (auto const& m : batch) { + if (m.type == TileLayerStream::MessageType::StringPool) { + stringPoolCommit = std::make_pair(nodeId, newOffset); + break; + } + } + } + + for (auto& m : batch) { + OutgoingFrame frame; + frame.bytes = std::move(m.bytes); + if (m.type == TileLayerStream::MessageType::StringPool) { + frame.stringPoolCommit = stringPoolCommit; + } + outgoing_.push_back(std::move(frame)); + } + } + + scheduleDrain(); + } + + void onRequestDone(size_t requestIndex, RequestStatus status) + { + if (cancelled_) + return; + + bool shouldEmit = false; + { + std::lock_guard lock(mutex_); + if (cancelled_) + return; + if (requestIndex >= requestStatuses_.size()) + return; + if (requestStatuses_[requestIndex] == status) + return; + requestStatuses_[requestIndex] = status; + shouldEmit = statusEmissionEnabled_; + } + + if (shouldEmit) { + queueStatusMessage({}); + scheduleDrain(); + } + } + + void queueStatusMessage(std::string message) + { + OutgoingFrame frame; + frame.bytes = encodeStreamMessage(TileLayerStream::MessageType::Status, buildStatusPayload(std::move(message))); + { + std::lock_guard lock(mutex_); + outgoing_.push_back(std::move(frame)); + } + } + + [[nodiscard]] std::string buildStatusPayload(std::string message) + { + nlohmann::json requestsJson = nlohmann::json::array(); + bool allDone = true; + + { + std::lock_guard lock(mutex_); + for (size_t i = 0; i < requests_.size(); ++i) { + const auto status = (i < requestStatuses_.size()) ? requestStatuses_[i] : RequestStatus::Open; + allDone &= (status != RequestStatus::Open); + + nlohmann::json reqJson = nlohmann::json::object(); + reqJson["index"] = i; + if (i < requests_.size() && requests_[i]) { + reqJson["mapId"] = requests_[i]->mapId_; + reqJson["layerId"] = requests_[i]->layerId_; + } else { + reqJson["mapId"] = ""; + reqJson["layerId"] = ""; + } + reqJson["status"] = static_cast>(status); + reqJson["statusText"] = std::string(requestStatusToString(status)); + requestsJson.push_back(std::move(reqJson)); + } + } + + return nlohmann::json::object({ + {"type", "mapget.tiles.status"}, + {"allDone", allDone}, + {"requests", std::move(requestsJson)}, + {"message", std::move(message)}, + }).dump(); + } + + void scheduleDrain() + { + if (drainScheduled_.exchange(true)) + return; + + auto weak = weak_from_this(); + loop_->queueInLoop([weak = std::move(weak)]() mutable { + if (auto self = weak.lock()) { + self->drainOnLoop(); + } + }); + } + + void drainOnLoop() + { + drainScheduled_ = false; + + auto conn = conn_.lock(); + if (!conn || conn->disconnected()) { + cancelNoStatus(); + return; + } + + constexpr size_t maxFramesPerDrain = 256; + for (size_t i = 0; i < maxFramesPerDrain; ++i) { + OutgoingFrame frame; + { + std::lock_guard lock(mutex_); + if (outgoing_.empty()) { + break; + } + frame = std::move(outgoing_.front()); + outgoing_.pop_front(); + } + + conn->send(frame.bytes, drogon::WebSocketMessageType::Binary); + if (frame.stringPoolCommit) { + if (auto state = connState_.lock()) { + state->stringPoolOffsets[frame.stringPoolCommit->first] = frame.stringPoolCommit->second; + } + } + } + + { + std::lock_guard lock(mutex_); + if (outgoing_.empty()) + return; + } + scheduleDrain(); + } + + HttpService& service_; + trantor::EventLoop* loop_; + std::weak_ptr conn_; + std::weak_ptr connState_; + + AuthHeaders authHeaders_; + + std::mutex mutex_; + std::deque outgoing_; + + std::vector requests_; + std::vector requestStatuses_; + bool statusEmissionEnabled_ = false; + + TileLayerStream::StringPoolOffsetMap offsets_; + std::unique_ptr writer_; + std::vector* currentWriteBatch_ = nullptr; + + std::atomic_bool drainScheduled_{false}; + std::atomic_bool cancelled_{false}; +}; + +class TilesWebSocketController final : public drogon::WebSocketController +{ +public: + explicit TilesWebSocketController(HttpService& service) : service_(service) {} + + void handleNewConnection(const drogon::HttpRequestPtr& req, const drogon::WebSocketConnectionPtr& conn) override + { + auto state = std::make_shared(); + state->authHeaders = authHeadersFromRequest(req); + conn->setContext(std::move(state)); + } + + void handleNewMessage( + const drogon::WebSocketConnectionPtr& conn, + std::string&& message, + const drogon::WebSocketMessageType& type) override + { + auto state = conn->getContext(); + if (!state) { + state = std::make_shared(); + conn->setContext(state); + } + + if (type != drogon::WebSocketMessageType::Text) { + const auto payload = nlohmann::json::object({ + {"type", "mapget.tiles.status"}, + {"allDone", true}, + {"requests", nlohmann::json::array()}, + {"message", "Expected a text message containing JSON."}, + }).dump(); + conn->send(encodeStreamMessage(TileLayerStream::MessageType::Status, payload), drogon::WebSocketMessageType::Binary); + return; + } + + nlohmann::json j; + try { + j = nlohmann::json::parse(message); + } + catch (const std::exception& e) { + const auto payload = nlohmann::json::object({ + {"type", "mapget.tiles.status"}, + {"allDone", true}, + {"requests", nlohmann::json::array()}, + {"message", fmt::format("Invalid JSON: {}", e.what())}, + }).dump(); + conn->send(encodeStreamMessage(TileLayerStream::MessageType::Status, payload), drogon::WebSocketMessageType::Binary); + return; + } + + // Patch per-connection string pool offsets if supplied. + if (j.contains("stringPoolOffsets")) { + if (!j["stringPoolOffsets"].is_object()) { + const auto payload = nlohmann::json::object({ + {"type", "mapget.tiles.status"}, + {"allDone", true}, + {"requests", nlohmann::json::array()}, + {"message", "stringPoolOffsets must be an object."}, + }).dump(); + conn->send(encodeStreamMessage(TileLayerStream::MessageType::Status, payload), drogon::WebSocketMessageType::Binary); + return; + } + try { + for (auto const& item : j["stringPoolOffsets"].items()) { + state->stringPoolOffsets[item.key()] = item.value().get(); + } + } + catch (const std::exception& e) { + const auto payload = nlohmann::json::object({ + {"type", "mapget.tiles.status"}, + {"allDone", true}, + {"requests", nlohmann::json::array()}, + {"message", fmt::format("Invalid stringPoolOffsets: {}", e.what())}, + }).dump(); + conn->send(encodeStreamMessage(TileLayerStream::MessageType::Status, payload), drogon::WebSocketMessageType::Binary); + return; + } + } + + if (state->session) { + state->session->cancel("Replaced by a new /tiles WebSocket request."); + state->session.reset(); + } + + state->session = std::make_shared( + service_, + conn, + state, + state->authHeaders, + state->stringPoolOffsets); + state->session->start(j); + } + + void handleConnectionClosed(const drogon::WebSocketConnectionPtr& conn) override + { + if (auto state = conn->getContext()) { + if (state->session) { + state->session->cancel("WebSocket connection closed."); + } + } + } + + WS_PATH_LIST_BEGIN + WS_PATH_ADD("/tiles", drogon::Get); + WS_PATH_LIST_END + +private: + HttpService& service_; +}; + +} // namespace + +void registerTilesWebSocketController(drogon::HttpAppFramework& app, HttpService& service) +{ + app.registerController(std::make_shared(service)); +} + +} // namespace mapget::detail diff --git a/libs/http-service/src/tiles-ws-controller.h b/libs/http-service/src/tiles-ws-controller.h new file mode 100644 index 00000000..4acaed45 --- /dev/null +++ b/libs/http-service/src/tiles-ws-controller.h @@ -0,0 +1,19 @@ +#pragma once + +namespace drogon +{ +class HttpAppFramework; +} + +namespace mapget +{ +class HttpService; +} + +namespace mapget::detail +{ + +void registerTilesWebSocketController(drogon::HttpAppFramework& app, HttpService& service); + +} // namespace mapget::detail + diff --git a/libs/model/include/mapget/model/stream.h b/libs/model/include/mapget/model/stream.h index 4568e117..5308ba4f 100644 --- a/libs/model/include/mapget/model/stream.h +++ b/libs/model/include/mapget/model/stream.h @@ -29,6 +29,12 @@ class TileLayerStream StringPool = 1, TileFeatureLayer = 2, TileSourceDataLayer = 3, + /** + * JSON-encoded status updates, e.g. for WebSocket /tiles. + * + * Payload: UTF-8 JSON bytes (not null-terminated). + */ + Status = 4, EndOfStream = 128 }; diff --git a/libs/model/src/stream.cpp b/libs/model/src/stream.cpp index f7d78c18..f59da85d 100644 --- a/libs/model/src/stream.cpp +++ b/libs/model/src/stream.cpp @@ -50,7 +50,6 @@ bool TileLayerStream::Reader::continueReading() } } - bitsery::Deserializer s(buffer_); auto numUnreadBytes = buffer_.tellp() - buffer_.tellg(); if (numUnreadBytes < nextValueSize_) return false; @@ -80,6 +79,12 @@ bool TileLayerStream::Reader::continueReading() std::string stringPoolNodeId = StringPool::readDataSourceNodeId(buffer_); stringPoolProvider_->getStringPool(stringPoolNodeId)->read(buffer_); } + else + { + // Skip unknown message types for forward compatibility (e.g. status + // messages on WebSocket streams). + buffer_.seekg(nextValueSize_, std::ios_base::cur); + } currentPhase_ = Phase::ReadHeader; return true; diff --git a/test/unit/test-http-datasource.cpp b/test/unit/test-http-datasource.cpp index a39f63c6..a2f12e2a 100644 --- a/test/unit/test-http-datasource.cpp +++ b/test/unit/test-http-datasource.cpp @@ -1,9 +1,11 @@ #include +#include #include #include #include #include +#include #include #include #include @@ -19,6 +21,7 @@ #include #include +#include #include #include "process.hpp" @@ -386,6 +389,296 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") REQUIRE(request->getStatus() == RequestStatus::Success); REQUIRE(receivedTileCount == 1); } + + auto runWsTilesRequest = [&](bool sendAuthHeader, std::string requestJson) { + auto wsLoopThread = std::make_unique("MapgetTestWsClient"); + wsLoopThread->run(); + + auto wsClient = drogon::WebSocketClient::newWebSocketClient( + fmt::format("ws://127.0.0.1:{}", service.port()), + wsLoopThread->getLoop()); + + std::mutex mutex; + std::condition_variable cv; + std::optional lastStatus; + std::atomic_int receivedTileCount{0}; + std::string error; + + const auto dsInfo = remoteDataSource->info(); + const auto layerInfo = dsInfo.getLayer("WayLayer"); + REQUIRE(layerInfo != nullptr); + + TileLayerStream::Reader reader( + [&](auto&&, auto&&) { return layerInfo; }, + [&](auto&& tile) { + if (tile->id().layer_ != LayerType::Features) { + std::lock_guard lock(mutex); + error = "Unexpected tile layer type"; + } + receivedTileCount.fetch_add(1, std::memory_order_relaxed); + }); + + wsClient->setMessageHandler( + [&](std::string&& msg, + const drogon::WebSocketClientPtr&, + const drogon::WebSocketMessageType& msgType) { + if (msgType != drogon::WebSocketMessageType::Binary) { + return; + } + + TileLayerStream::MessageType type = TileLayerStream::MessageType::None; + uint32_t payloadSize = 0; + std::stringstream ss; + ss.write(msg.data(), static_cast(msg.size())); + if (!TileLayerStream::Reader::readMessageHeader(ss, type, payloadSize)) { + std::lock_guard lock(mutex); + error = "Failed to read stream message header"; + cv.notify_all(); + return; + } + + if (type == TileLayerStream::MessageType::Status) { + std::string payload(payloadSize, '\0'); + ss.read(payload.data(), static_cast(payloadSize)); + nlohmann::json parsed; + try { + parsed = nlohmann::json::parse(payload); + } + catch (const std::exception& e) { + std::lock_guard lock(mutex); + error = std::string("Failed to parse status JSON: ") + e.what(); + cv.notify_all(); + return; + } + { + std::lock_guard lock(mutex); + lastStatus = std::move(parsed); + } + cv.notify_all(); + return; + } + + try { + reader.read(msg); + } + catch (const std::exception& e) { + std::lock_guard lock(mutex); + error = std::string("Failed to parse tile stream: ") + e.what(); + cv.notify_all(); + } + }); + + auto connectReq = drogon::HttpRequest::newHttpRequest(); + connectReq->setMethod(drogon::Get); + connectReq->setPath("/tiles"); + if (sendAuthHeader) { + connectReq->addHeader("X-USER-ROLE", "Tropico-Viewer"); + } + + std::promise connectPromise; + auto connectFuture = connectPromise.get_future(); + wsClient->connectToServer( + connectReq, + [&connectPromise]( + drogon::ReqResult result, + const drogon::HttpResponsePtr&, + const drogon::WebSocketClientPtr&) { connectPromise.set_value(result); }); + + REQUIRE(connectFuture.wait_for(std::chrono::seconds(5)) == std::future_status::ready); + REQUIRE(connectFuture.get() == drogon::ReqResult::Ok); + + auto conn = wsClient->getConnection(); + if (!conn || !conn->connected()) { + wsClient->stop(); + FAIL("WebSocket connection not established"); + } + + conn->send(requestJson, drogon::WebSocketMessageType::Text); + + { + std::unique_lock lock(mutex); + REQUIRE(cv.wait_for(lock, std::chrono::seconds(10), [&] { + return !error.empty() || + (lastStatus.has_value() && lastStatus->value("allDone", false)); + })); + if (!error.empty()) { + wsClient->stop(); + FAIL(error); + } + } + + wsClient->stop(); + + REQUIRE(lastStatus.has_value()); + return std::make_tuple(*lastStatus, receivedTileCount.load(std::memory_order_relaxed)); + }; + + // WebSocket tiles: unauthorized without auth header. + { + auto req = nlohmann::json::object({ + {"requests", nlohmann::json::array({nlohmann::json::object({ + {"mapId", "Tropico"}, + {"layerId", "WayLayer"}, + {"tileIds", nlohmann::json::array({1234})}, + })})}, + }).dump(); + + auto [status, wsTileCount] = runWsTilesRequest(false, req); + REQUIRE(wsTileCount == 0); + REQUIRE(status["requests"].size() == 1); + REQUIRE(status["requests"][0]["status"].get() == + static_cast(RequestStatus::Unauthorized)); + } + + // WebSocket tiles: invalid request stays on the same connection, then succeeds. + { + auto wsLoopThread = std::make_unique("MapgetTestWsClientReuse"); + wsLoopThread->run(); + + auto wsClient = drogon::WebSocketClient::newWebSocketClient( + fmt::format("ws://127.0.0.1:{}", service.port()), + wsLoopThread->getLoop()); + + std::mutex mutex; + std::condition_variable cv; + std::optional lastStatus; + std::atomic_int receivedTileCount{0}; + std::string error; + + const auto dsInfo = remoteDataSource->info(); + const auto layerInfo = dsInfo.getLayer("WayLayer"); + REQUIRE(layerInfo != nullptr); + + TileLayerStream::Reader reader( + [&](auto&&, auto&&) { return layerInfo; }, + [&](auto&&) { receivedTileCount.fetch_add(1, std::memory_order_relaxed); }); + + wsClient->setMessageHandler( + [&](std::string&& msg, + const drogon::WebSocketClientPtr&, + const drogon::WebSocketMessageType& msgType) { + if (msgType != drogon::WebSocketMessageType::Binary) { + return; + } + + TileLayerStream::MessageType type = TileLayerStream::MessageType::None; + uint32_t payloadSize = 0; + std::stringstream ss; + ss.write(msg.data(), static_cast(msg.size())); + if (!TileLayerStream::Reader::readMessageHeader(ss, type, payloadSize)) { + std::lock_guard lock(mutex); + error = "Failed to read stream message header"; + cv.notify_all(); + return; + } + + if (type == TileLayerStream::MessageType::Status) { + std::string payload(payloadSize, '\0'); + ss.read(payload.data(), static_cast(payloadSize)); + nlohmann::json parsed; + try { + parsed = nlohmann::json::parse(payload); + } + catch (const std::exception& e) { + std::lock_guard lock(mutex); + error = std::string("Failed to parse status JSON: ") + e.what(); + cv.notify_all(); + return; + } + { + std::lock_guard lock(mutex); + lastStatus = std::move(parsed); + } + cv.notify_all(); + return; + } + + try { + reader.read(msg); + } + catch (const std::exception& e) { + std::lock_guard lock(mutex); + error = std::string("Failed to parse tile stream: ") + e.what(); + cv.notify_all(); + } + }); + + auto connectReq = drogon::HttpRequest::newHttpRequest(); + connectReq->setMethod(drogon::Get); + connectReq->setPath("/tiles"); + connectReq->addHeader("X-USER-ROLE", "Tropico-Viewer"); + + std::promise connectPromise; + auto connectFuture = connectPromise.get_future(); + wsClient->connectToServer( + connectReq, + [&connectPromise]( + drogon::ReqResult result, + const drogon::HttpResponsePtr&, + const drogon::WebSocketClientPtr&) { connectPromise.set_value(result); }); + + REQUIRE(connectFuture.wait_for(std::chrono::seconds(5)) == std::future_status::ready); + REQUIRE(connectFuture.get() == drogon::ReqResult::Ok); + + auto conn = wsClient->getConnection(); + if (!conn || !conn->connected()) { + wsClient->stop(); + FAIL("WebSocket connection not established"); + } + + // Invalid JSON: should yield a Status message but keep the socket open. + { + conn->send("{not json", drogon::WebSocketMessageType::Text); + std::unique_lock lock(mutex); + REQUIRE(cv.wait_for(lock, std::chrono::seconds(5), [&] { + return !error.empty() || + (lastStatus.has_value() && lastStatus->value("allDone", false)); + })); + if (!error.empty()) { + wsClient->stop(); + FAIL(error); + } + REQUIRE(lastStatus->value("message", "").find("Invalid JSON") != std::string::npos); + REQUIRE(conn->connected()); + } + + // Valid request should succeed afterwards. + { + { + std::lock_guard lock(mutex); + lastStatus.reset(); + } + receivedTileCount.store(0, std::memory_order_relaxed); + + auto req = nlohmann::json::object({ + {"requests", nlohmann::json::array({nlohmann::json::object({ + {"mapId", "Tropico"}, + {"layerId", "WayLayer"}, + {"tileIds", nlohmann::json::array({1234})}, + })})}, + }).dump(); + + conn->send(req, drogon::WebSocketMessageType::Text); + + std::unique_lock lock(mutex); + REQUIRE(cv.wait_for(lock, std::chrono::seconds(10), [&] { + return !error.empty() || + (lastStatus.has_value() && lastStatus->value("allDone", false)); + })); + if (!error.empty()) { + wsClient->stop(); + FAIL(error); + } + + REQUIRE(receivedTileCount.load(std::memory_order_relaxed) == 1); + REQUIRE(lastStatus->contains("requests")); + REQUIRE((*lastStatus)["requests"].size() == 1); + REQUIRE((*lastStatus)["requests"][0]["status"].get() == + static_cast(RequestStatus::Success)); + } + + wsClient->stop(); + } } service.remove(remoteDataSource); From 9d36ecf86728a0751b6ac1f3c7b4b9d6f55712f8 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Tue, 27 Jan 2026 14:21:41 +0100 Subject: [PATCH 13/38] Communicate Tile Load State. --- .../http-datasource/datasource-client.h | 12 ++++- .../http-datasource/src/datasource-client.cpp | 19 ++++++-- libs/http-datasource/src/http-server.cpp | 2 +- libs/http-service/src/cli.cpp | 2 +- libs/http-service/src/tiles-ws-controller.cpp | 46 +++++++++++++++++++ libs/model/include/mapget/model/layer.h | 17 +++++++ libs/model/include/mapget/model/stream.h | 8 +++- libs/model/src/layer.cpp | 12 +++++ .../include/mapget/service/datasource.h | 6 ++- libs/service/include/mapget/service/service.h | 8 ++++ libs/service/src/datasource.cpp | 12 ++++- libs/service/src/service.cpp | 17 ++++++- 12 files changed, 149 insertions(+), 12 deletions(-) diff --git a/libs/http-datasource/include/mapget/http-datasource/datasource-client.h b/libs/http-datasource/include/mapget/http-datasource/datasource-client.h index 91038d73..5a00d000 100644 --- a/libs/http-datasource/include/mapget/http-datasource/datasource-client.h +++ b/libs/http-datasource/include/mapget/http-datasource/datasource-client.h @@ -47,7 +47,11 @@ class RemoteDataSource : public DataSource DataSourceInfo info() override; void fill(TileFeatureLayer::Ptr const& featureTile) override; void fill(TileSourceDataLayer::Ptr const& blobTile) override; - TileLayer::Ptr get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info) override; + TileLayer::Ptr get( + MapTileKey const& k, + Cache::Ptr& cache, + DataSourceInfo const& info, + TileLayer::LoadStateCallback loadStateCallback = {}) override; std::vector locate(const mapget::LocateRequest &req) override; private: @@ -87,7 +91,11 @@ class RemoteDataSourceProcess : public DataSource DataSourceInfo info() override; void fill(TileFeatureLayer::Ptr const& featureTile) override; void fill(TileSourceDataLayer::Ptr const& sourceDataLayer) override; - TileLayer::Ptr get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info) override; + TileLayer::Ptr get( + MapTileKey const& k, + Cache::Ptr& cache, + DataSourceInfo const& info, + TileLayer::LoadStateCallback loadStateCallback = {}) override; std::vector locate(const mapget::LocateRequest &req) override; private: diff --git a/libs/http-datasource/src/datasource-client.cpp b/libs/http-datasource/src/datasource-client.cpp index 128a5446..0a17088b 100644 --- a/libs/http-datasource/src/datasource-client.cpp +++ b/libs/http-datasource/src/datasource-client.cpp @@ -70,7 +70,11 @@ void RemoteDataSource::fill(const TileSourceDataLayer::Ptr& blobTile) } TileLayer::Ptr -RemoteDataSource::get(const MapTileKey& k, Cache::Ptr& cache, const DataSourceInfo& info) +RemoteDataSource::get( + const MapTileKey& k, + Cache::Ptr& cache, + const DataSourceInfo& info, + TileLayer::LoadStateCallback loadStateCallback) { // Round-robin usage of http clients to facilitate parallel requests. auto& client = httpClients_[(nextClient_++) % httpClients_.size()]; @@ -101,7 +105,7 @@ RemoteDataSource::get(const MapTileKey& k, Cache::Ptr& cache, const DataSourceIn // Use tile instantiation logic of the base class, // the error is then set in fill(). - return DataSource::get(k, cache, info); + return DataSource::get(k, cache, info, std::move(loadStateCallback)); } // Check the response body for expected content. @@ -112,6 +116,9 @@ RemoteDataSource::get(const MapTileKey& k, Cache::Ptr& cache, const DataSourceIn cache); reader.read(std::string(tileResponse->body())); + if (result && loadStateCallback) { + result->setLoadStateCallback(std::move(loadStateCallback)); + } return result; } @@ -243,11 +250,15 @@ void RemoteDataSourceProcess::fill(TileSourceDataLayer::Ptr const& sourceDataLay } TileLayer::Ptr -RemoteDataSourceProcess::get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info) +RemoteDataSourceProcess::get( + MapTileKey const& k, + Cache::Ptr& cache, + DataSourceInfo const& info, + TileLayer::LoadStateCallback loadStateCallback) { if (!remoteSource_) raise("Remote data source is not initialized."); - return remoteSource_->get(k, cache, info); + return remoteSource_->get(k, cache, info, std::move(loadStateCallback)); } std::vector RemoteDataSourceProcess::locate(const LocateRequest& req) diff --git a/libs/http-datasource/src/http-server.cpp b/libs/http-datasource/src/http-server.cpp index bd917360..422ab1b6 100644 --- a/libs/http-datasource/src/http-server.cpp +++ b/libs/http-datasource/src/http-server.cpp @@ -265,7 +265,7 @@ bool HttpServer::mountFileSystem(std::string const& pathFromTo) if (!exists || ec) return false; auto isDirectory = std::filesystem::is_directory(fsRoot, ec); - if (isDirectory || ec) + if (!isDirectory || ec) return false; std::scoped_lock lock(impl_->mountsMutex_); diff --git a/libs/http-service/src/cli.cpp b/libs/http-service/src/cli.cpp index 7e51c540..f8ea8c6a 100644 --- a/libs/http-service/src/cli.cpp +++ b/libs/http-service/src/cli.cpp @@ -446,7 +446,7 @@ struct ServeCommand log().info("Webapp: {}", webapp_); if (!srv.mountFileSystem(webapp_)) { log().error(" ...failed to mount!"); - exit(1); + raise("Failed to mount webapp filesystem path."); } } diff --git a/libs/http-service/src/tiles-ws-controller.cpp b/libs/http-service/src/tiles-ws-controller.cpp index 82fefac9..6d7f5d23 100644 --- a/libs/http-service/src/tiles-ws-controller.cpp +++ b/libs/http-service/src/tiles-ws-controller.cpp @@ -58,6 +58,19 @@ namespace return "Unknown"; } +[[nodiscard]] std::string_view loadStateToString(TileLayer::LoadState s) +{ + switch (s) { + case TileLayer::LoadState::LoadingQueued: + return "LoadingQueued"; + case TileLayer::LoadState::BackendFetching: + return "BackendFetching"; + case TileLayer::LoadState::BackendConverting: + return "BackendConverting"; + } + return "Unknown"; +} + [[nodiscard]] std::string encodeStreamMessage(TileLayerStream::MessageType type, std::string_view payload) { std::ostringstream headerStream; @@ -163,6 +176,11 @@ class TilesWsSession : public std::enable_shared_from_this self->onTileLayer(std::move(layer)); } }); + req->onLayerLoadStateChanged([weak](MapTileKey const& key, TileLayer::LoadState state) { + if (auto self = weak.lock()) { + self->onLoadStateChanged(key, state); + } + }); req->onDone_ = [weak, i](RequestStatus status) { if (auto self = weak.lock()) { self->onRequestDone(i, status); @@ -332,6 +350,22 @@ class TilesWsSession : public std::enable_shared_from_this } } + void onLoadStateChanged(MapTileKey const& key, TileLayer::LoadState state) + { + if (cancelled_) + return; + + OutgoingFrame frame; + frame.bytes = encodeStreamMessage( + TileLayerStream::MessageType::LoadStateChange, + buildLoadStatePayload(key, state)); + { + std::lock_guard lock(mutex_); + outgoing_.push_back(std::move(frame)); + } + scheduleDrain(); + } + [[nodiscard]] std::string buildStatusPayload(std::string message) { nlohmann::json requestsJson = nlohmann::json::array(); @@ -366,6 +400,18 @@ class TilesWsSession : public std::enable_shared_from_this }).dump(); } + [[nodiscard]] std::string buildLoadStatePayload(MapTileKey const& key, TileLayer::LoadState state) + { + return nlohmann::json::object({ + {"type", "mapget.tiles.load-state"}, + {"mapId", key.mapId_}, + {"layerId", key.layerId_}, + {"tileId", key.tileId_.value_}, + {"state", static_cast(state)}, + {"stateText", std::string(loadStateToString(state))}, + }).dump(); + } + void scheduleDrain() { if (drainScheduled_.exchange(true)) diff --git a/libs/model/include/mapget/model/layer.h b/libs/model/include/mapget/model/layer.h index 5a2acfa5..0f383fa4 100644 --- a/libs/model/include/mapget/model/layer.h +++ b/libs/model/include/mapget/model/layer.h @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace simfil { struct StringPool; } @@ -85,6 +86,12 @@ class TileLayer { public: using Ptr = std::shared_ptr; + enum class LoadState : uint8_t { + LoadingQueued = 0, + BackendFetching = 1, + BackendConverting = 2 + }; + using LoadStateCallback = std::function; /** * Constructor that takes tileId_, nodeId_, mapId_, layerInfo_, @@ -192,6 +199,15 @@ class TileLayer virtual tl::expected write(std::ostream& outputStream); virtual nlohmann::json toJson() const; + /** + * Set a load-state callback. Used by the service to forward state changes. + * Not serialized with the tile. + */ + void setLoadStateCallback(LoadStateCallback cb); + + /** Emit a load-state change (if a callback is registered). */ + void setLoadState(LoadState state); + protected: Version mapVersion_{0, 0, 0}; TileId tileId_; @@ -204,6 +220,7 @@ class TileLayer std::optional ttl_; nlohmann::json info_; std::optional legalInfo_; // Copyright-related information + LoadStateCallback onLoadStateChanged_; }; } diff --git a/libs/model/include/mapget/model/stream.h b/libs/model/include/mapget/model/stream.h index 5308ba4f..3d9dae76 100644 --- a/libs/model/include/mapget/model/stream.h +++ b/libs/model/include/mapget/model/stream.h @@ -35,13 +35,19 @@ class TileLayerStream * Payload: UTF-8 JSON bytes (not null-terminated). */ Status = 4, + /** + * JSON-encoded load-state updates for individual tiles. + * + * Payload: UTF-8 JSON bytes (not null-terminated). + */ + LoadStateChange = 5, EndOfStream = 128 }; struct StringPoolCache; /** Protocol Version which parsed blobs must be compatible with. */ - static constexpr Version CurrentProtocolVersion{0, 1, 1}; + static constexpr Version CurrentProtocolVersion{1, 0, 0}; /** Map to keep track of the highest sent string id per datasource node. */ using StringPoolOffsetMap = std::unordered_map; diff --git a/libs/model/src/layer.cpp b/libs/model/src/layer.cpp index be94dbac..199851c7 100644 --- a/libs/model/src/layer.cpp +++ b/libs/model/src/layer.cpp @@ -243,6 +243,18 @@ void TileLayer::setLegalInfo(const std::string& legalInfoString) legalInfo_ = legalInfoString; } +void TileLayer::setLoadStateCallback(LoadStateCallback cb) +{ + onLoadStateChanged_ = std::move(cb); +} + +void TileLayer::setLoadState(LoadState state) +{ + if (onLoadStateChanged_) { + onLoadStateChanged_(state); + } +} + tl::expected TileLayer::write(std::ostream& outputStream) { using namespace std::chrono; diff --git a/libs/service/include/mapget/service/datasource.h b/libs/service/include/mapget/service/datasource.h index a76c172e..dadf5ae7 100644 --- a/libs/service/include/mapget/service/datasource.h +++ b/libs/service/include/mapget/service/datasource.h @@ -58,7 +58,11 @@ class DataSource virtual std::vector locate(LocateRequest const& req); /** Called by mapget::Service worker. Dispatches to Cache or fill(...) on miss. */ - virtual TileLayer::Ptr get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info); + virtual TileLayer::Ptr get( + MapTileKey const& k, + Cache::Ptr& cache, + DataSourceInfo const& info, + TileLayer::LoadStateCallback loadStateCallback = {}); /** Add an authorization header-regex pair for this datasource. */ void requireAuthHeaderRegexMatchOption(std::string header, std::regex re); diff --git a/libs/service/include/mapget/service/service.h b/libs/service/include/mapget/service/service.h index 79c9689e..34789a5d 100644 --- a/libs/service/include/mapget/service/service.h +++ b/libs/service/include/mapget/service/service.h @@ -76,8 +76,15 @@ class LayerTilesRequest template LayerTilesRequest& onSourceDataLayer(Fun&& callback) { onSourceDataLayer_ = std::forward(callback); return *this; } + /** + * Callback for per-tile load-state changes. + */ + template + LayerTilesRequest& onLayerLoadStateChanged(Fun&& callback) { onLoadStateChanged_ = std::forward(callback); return *this; } + protected: virtual void notifyResult(TileLayer::Ptr); + void notifyLoadState(MapTileKey const& key, TileLayer::LoadState state); void setStatus(RequestStatus s); void notifyStatus(); nlohmann::json toJson(); @@ -88,6 +95,7 @@ class LayerTilesRequest */ std::function onFeatureLayer_; std::function onSourceDataLayer_; + std::function onLoadStateChanged_; // So the service can track which tileId index from tiles_ // is next in line to be processed. diff --git a/libs/service/src/datasource.cpp b/libs/service/src/datasource.cpp index 8074fcd4..3e7715fe 100644 --- a/libs/service/src/datasource.cpp +++ b/libs/service/src/datasource.cpp @@ -10,7 +10,11 @@ namespace mapget { -TileLayer::Ptr DataSource::get(const MapTileKey& k, Cache::Ptr& cache, DataSourceInfo const& info) +TileLayer::Ptr DataSource::get( + const MapTileKey& k, + Cache::Ptr& cache, + DataSourceInfo const& info, + TileLayer::LoadStateCallback loadStateCallback) { auto layerInfo = info.getLayer(k.layerId_); if (!layerInfo) @@ -27,6 +31,9 @@ TileLayer::Ptr DataSource::get(const MapTileKey& k, Cache::Ptr& cache, DataSourc info.mapId_, info.getLayer(k.layerId_), cache->getStringPool(info.nodeId_)); + if (loadStateCallback) { + tileFeatureLayer->setLoadStateCallback(loadStateCallback); + } fill(tileFeatureLayer); result = tileFeatureLayer; break; @@ -38,6 +45,9 @@ TileLayer::Ptr DataSource::get(const MapTileKey& k, Cache::Ptr& cache, DataSourc info.mapId_, info.getLayer(k.layerId_), cache->getStringPool(info.nodeId_)); + if (loadStateCallback) { + tileSourceDataLayer->setLoadStateCallback(loadStateCallback); + } fill(tileSourceDataLayer); result = tileSourceDataLayer; break; diff --git a/libs/service/src/service.cpp b/libs/service/src/service.cpp index 4abbf9bd..4c1c5678 100644 --- a/libs/service/src/service.cpp +++ b/libs/service/src/service.cpp @@ -62,6 +62,13 @@ void LayerTilesRequest::notifyResult(TileLayer::Ptr r) { } } +void LayerTilesRequest::notifyLoadState(MapTileKey const& key, TileLayer::LoadState state) +{ + if (onLoadStateChanged_) { + onLoadStateChanged_(key, state); + } +} + void LayerTilesRequest::setStatus(RequestStatus s) { { @@ -188,6 +195,7 @@ struct Service::Controller // Enter into the jobs-in-progress set. jobsInProgress_.insert(result->tileKey); + request->notifyLoadState(result->tileKey, TileLayer::LoadState::LoadingQueued); // Move this request to the end of the list, so others gain priority. requests_.splice(requests_.end(), requests_, reqIt); @@ -262,7 +270,14 @@ struct Service::Worker dataSource_->onCacheExpired(job.tileKey, *job.cacheExpiredAt); } - auto layer = dataSource_->get(job.tileKey, controller_.cache_, info_); + job.request->notifyLoadState(job.tileKey, TileLayer::LoadState::BackendFetching); + auto layer = dataSource_->get( + job.tileKey, + controller_.cache_, + info_, + [request = job.request, tileKey = job.tileKey](TileLayer::LoadState state) { + request->notifyLoadState(tileKey, state); + }); if (!layer) raise("DataSource::get() returned null."); From 87acbb451ab509820a3771b921e7b0e4d6220832 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 29 Jan 2026 17:19:49 +0100 Subject: [PATCH 14/38] Ensure that work on a tile always benefit all interested requests immediately. Bump protocol version. --- libs/model/include/mapget/model/stream.h | 16 +- libs/service/include/mapget/service/service.h | 4 + libs/service/src/service.cpp | 188 +++++++++++------- 3 files changed, 133 insertions(+), 75 deletions(-) diff --git a/libs/model/include/mapget/model/stream.h b/libs/model/include/mapget/model/stream.h index 3d9dae76..5eb0a5a5 100644 --- a/libs/model/include/mapget/model/stream.h +++ b/libs/model/include/mapget/model/stream.h @@ -46,8 +46,20 @@ class TileLayerStream struct StringPoolCache; - /** Protocol Version which parsed blobs must be compatible with. */ - static constexpr Version CurrentProtocolVersion{1, 0, 0}; + /** + * Protocol Version which parsed blobs must be compatible with. + * Version History: + * - Version 1.0: + * + Added TileFeatureLayer Message + * + Added StringPool Message + * + Added TileSourceDataLayer Message + * + Added EndOfStream Message + * - Version 1.1: + * + Added errorCode field to TileLayer + * + Added Status Message + * + Added LoadStateChange Message + */ + static constexpr Version CurrentProtocolVersion{1, 1, 0}; /** Map to keep track of the highest sent string id per datasource node. */ using StringPoolOffsetMap = std::unordered_map; diff --git a/libs/service/include/mapget/service/service.h b/libs/service/include/mapget/service/service.h index 34789a5d..c772dd33 100644 --- a/libs/service/include/mapget/service/service.h +++ b/libs/service/include/mapget/service/service.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace mapget { @@ -101,6 +102,9 @@ class LayerTilesRequest // is next in line to be processed. size_t nextTileIndex_ = 0; + // Track which tiles still need to be scheduled/served for this request. + std::set tileIdsNotDone_; + // So the requester can track how many results have been received. size_t resultCount_ = 0; diff --git a/libs/service/src/service.cpp b/libs/service/src/service.cpp index 4c1c5678..d645691f 100644 --- a/libs/service/src/service.cpp +++ b/libs/service/src/service.cpp @@ -11,13 +11,11 @@ #include #include -#include #include #include #include #include #include -#include #include #include "simfil/types.h" @@ -33,6 +31,7 @@ LayerTilesRequest::LayerTilesRequest( layerId_(std::move(layerId)), tiles_(std::move(tiles)) { + tileIdsNotDone_.insert(tiles_.begin(), tiles_.end()); if (tiles_.empty()) { // An empty request is always set to success, but the client/service // is responsible for triggering notifyStatus() in that case. @@ -43,16 +42,16 @@ LayerTilesRequest::LayerTilesRequest( void LayerTilesRequest::notifyResult(TileLayer::Ptr r) { const auto type = r->layerInfo()->type_; switch (type) { - case mapget::LayerType::Features: + case LayerType::Features: if (onFeatureLayer_) - onFeatureLayer_(std::move(std::static_pointer_cast(r))); + onFeatureLayer_(std::move(std::static_pointer_cast(r))); break; - case mapget::LayerType::SourceData: + case LayerType::SourceData: if (onSourceDataLayer_) - onSourceDataLayer_(std::move(std::static_pointer_cast(r))); + onSourceDataLayer_(std::move(std::static_pointer_cast(r))); break; default: - mapget::log().error(fmt::format("Unhandled layer type {}, no matching callback!", static_cast(type))); + log().error(fmt::format("Unhandled layer type {}, no matching callback!", static_cast(type))); break; } @@ -121,13 +120,15 @@ bool LayerTilesRequest::isDone() struct Service::Controller { + virtual ~Controller() = default; + struct Job { MapTileKey tileKey; - LayerTilesRequest::Ptr request; + std::vector waitingRequests; std::optional cacheExpiredAt; }; - std::set jobsInProgress_; // Set of jobs currently in progress + std::map> jobsInProgress_; // Jobs currently in progress + interested requests Cache::Ptr cache_; // The cache for the service std::optional defaultTtl_; // Default TTL applied when datasource does not override std::list requests_; // List of requests currently being processed @@ -136,79 +137,101 @@ struct Service::Controller explicit Controller(Cache::Ptr cache, std::optional defaultTtl) : cache_(std::move(cache)), - defaultTtl_(std::move(defaultTtl)) + defaultTtl_(defaultTtl) { if (!cache_) raise("Cache must not be null!"); } - std::optional nextJob(DataSourceInfo const& i) + std::shared_ptr nextJob(DataSourceInfo const& i) { // Workers call the nextJob function when they are free. // Note: For thread safety, jobsMutex_ must be held // when calling this function. - std::optional result; + std::shared_ptr result; // Return next job, if available. - bool cachedTilesServed = false; + bool cachedTilesServedOrInProgressSkipped = false; + bool anyTasksRemaining = false; do { - cachedTilesServed = false; + cachedTilesServedOrInProgressSkipped = false; + anyTasksRemaining = false; for (auto reqIt = requests_.begin(); reqIt != requests_.end(); ++reqIt) { auto const& request = *reqIt; auto layerIt = i.layers_.find(request->layerId_); - // Are there tiles left to be processed in the request? - if (request->mapId_ == i.mapId_ && layerIt != i.layers_.end()) { - if (request->nextTileIndex_ >= request->tiles_.size()) { - continue; + // Does the Datasource Info (i) of the worker fit the request? + if (request->mapId_ != i.mapId_ || layerIt == i.layers_.end()) + continue; + + // Find the next pending tile in the request's ordered list. + TileId tileId{}; + bool foundTile = false; + while (request->nextTileIndex_ < request->tiles_.size()) { + tileId = request->tiles_[request->nextTileIndex_++]; + if (request->tileIdsNotDone_.find(tileId) != request->tileIdsNotDone_.end()) { + foundTile = true; + break; } + } + if (!foundTile) + continue; + anyTasksRemaining = true; + auto resultTileKey = MapTileKey(layerIt->second->type_, request->mapId_, request->layerId_, tileId); + + // Cache lookup. + auto cachedResult = cache_->getTileLayer(resultTileKey, i); + if (cachedResult.tile) { + request->tileIdsNotDone_.erase(tileId); + log().debug("Serving cached tile: {}", resultTileKey.toString()); + request->notifyResult(cachedResult.tile); + cachedTilesServedOrInProgressSkipped = true; + continue; + } + + // If another worker is working on this tile, ensure that this request gets it as well. + if (auto inProgress = jobsInProgress_.find(resultTileKey); + inProgress != jobsInProgress_.end()) { + // This tile is already being processed. Register interest so the result + // can satisfy multiple requests, and allow this request to advance. + log().debug("Joining tile with job in progress: {}", + resultTileKey.toString()); + request->tileIdsNotDone_.erase(tileId); + inProgress->second->waitingRequests.push_back(request); + cachedTilesServedOrInProgressSkipped = true; + continue; + } - // Create result wrapper object. - auto tileId = request->tiles_[request->nextTileIndex_++]; - result = Job{MapTileKey(), request, std::nullopt}; - result->tileKey.layer_ = layerIt->second->type_; - result->tileKey.mapId_ = request->mapId_; - result->tileKey.layerId_ = request->layerId_; - result->tileKey.tileId_ = tileId; - - // Cache lookup. - auto cachedResult = cache_->getTileLayer(result->tileKey, i); - if (cachedResult.tile) { - log().debug("Serving cached tile: {}", result->tileKey.toString()); - request->notifyResult(cachedResult.tile); - result.reset(); - cachedTilesServed = true; + // We found something to work on that is not cached and not in progress - + // enter it into the jobs-in-progress map with the requesting client. + request->tileIdsNotDone_.erase(tileId); + result = std::make_shared(Job{resultTileKey, {request}, cachedResult.expiredAt}); + // Proactively attach other requests that need this tile. + for (auto const& otherRequest : requests_) { + if (!otherRequest || otherRequest == request) continue; - } - result->cacheExpiredAt = cachedResult.expiredAt; - - if (jobsInProgress_.find(result->tileKey) != jobsInProgress_.end()) { - // Don't work on something that is already being worked on. - // Wait for the work to finish, then send the (hopefully cached) result. - log().debug("Delaying tile with job in progress: {}", - result->tileKey.toString()); - --request->nextTileIndex_; - result.reset(); + if (otherRequest->mapId_ != request->mapId_ || otherRequest->layerId_ != request->layerId_) continue; - } - - // Enter into the jobs-in-progress set. - jobsInProgress_.insert(result->tileKey); - request->notifyLoadState(result->tileKey, TileLayer::LoadState::LoadingQueued); + if (otherRequest->tileIdsNotDone_.erase(tileId) == 0) + continue; + result->waitingRequests.push_back(otherRequest); + otherRequest->notifyLoadState(result->tileKey, TileLayer::LoadState::LoadingQueued); + } + jobsInProgress_.emplace(result->tileKey, result); + request->notifyLoadState(result->tileKey, TileLayer::LoadState::LoadingQueued); - // Move this request to the end of the list, so others gain priority. - requests_.splice(requests_.end(), requests_, reqIt); + // Move this request to the end of the list, so others gain priority. + requests_.splice(requests_.end(), requests_, reqIt); - log().debug("Working on tile: {}", result->tileKey.toString()); - break; - } + log().debug("Working on tile: {}", result->tileKey.toString()); + break; } } - while (cachedTilesServed && !result); + while (cachedTilesServedOrInProgressSkipped && !result && anyTasksRemaining); // Clean up done requests. - requests_.remove_if([](auto&& r) {return r->nextTileIndex_ == r->tiles_.size(); }); + requests_.remove_if([](auto&& r) {return r->tileIdsNotDone_.empty(); }); return result; } @@ -239,7 +262,7 @@ struct Service::Worker bool work() { - std::optional nextJob; + std::shared_ptr nextJob; { std::unique_lock lock(controller_.jobsMutex_); @@ -255,7 +278,7 @@ struct Service::Worker return true; } nextJob = controller_.nextJob(info_); - return nextJob.has_value(); + return !!nextJob; }); } @@ -270,13 +293,26 @@ struct Service::Worker dataSource_->onCacheExpired(job.tileKey, *job.cacheExpiredAt); } - job.request->notifyLoadState(job.tileKey, TileLayer::LoadState::BackendFetching); + auto notifyWaitingRequests = [&](TileLayer::LoadState state) { + std::vector waiting; + { + std::unique_lock lock(controller_.jobsMutex_); + waiting = job.waitingRequests; + } + for (auto const& req : waiting) { + if (req) { + req->notifyLoadState(job.tileKey, state); + } + } + }; + + notifyWaitingRequests(TileLayer::LoadState::BackendFetching); auto layer = dataSource_->get( job.tileKey, controller_.cache_, info_, - [request = job.request, tileKey = job.tileKey](TileLayer::LoadState state) { - request->notifyLoadState(tileKey, state); + [¬ifyWaitingRequests](TileLayer::LoadState state) { + notifyWaitingRequests(state); }); if (!layer) raise("DataSource::get() returned null."); @@ -299,14 +335,20 @@ struct Service::Worker controller_.cache_->putTileLayer(layer); + std::vector notifyRequests; { std::unique_lock lock(controller_.jobsMutex_); controller_.jobsInProgress_.erase(job.tileKey); - job.request->notifyResult(layer); - // As we entered a tile into the cache, notify other workers - // that this tile can be served. - controller_.jobsAvailable_.notify_all(); + notifyRequests = job.waitingRequests; + } + for (auto const& req : notifyRequests) { + if (req) { + req->notifyResult(layer); + } } + // As we entered a tile into the cache, notify other workers + // that this tile can be served. + controller_.jobsAvailable_.notify_all(); } catch (std::exception& e) { log().error("Could not load tile {}: {}", @@ -331,7 +373,7 @@ struct Service::Impl : public Service::Controller Cache::Ptr cache, bool useDataSourceConfig, std::optional defaultTtl) - : Controller(std::move(cache), std::move(defaultTtl)) + : Controller(std::move(cache), defaultTtl) { if (!useDataSourceConfig) return; @@ -361,7 +403,7 @@ struct Service::Impl : public Service::Controller }); } - ~Impl() + ~Impl() override { // Ensure that no new datasources are added while we are cleaning up. configSubscription_.reset(); @@ -490,21 +532,21 @@ struct Service::Impl : public Service::Controller if (auxDataSource->info().mapId_ == baseTile->mapId()) { auto auxTile = [&]() -> TileFeatureLayer::Ptr { - auto auxTile = auxDataSource->get(baseTile->id(), cache_, auxDataSource->info()); - if (!auxTile) { + auto result = auxDataSource->get(baseTile->id(), cache_, auxDataSource->info()); + if (!result) { log().warn("auxDataSource returned null for {}", baseTile->id().toString()); return {}; } - if (auxTile->error()) { - log().warn("Error while fetching addon tile {}: {}", baseTile->id().toString(), *auxTile->error()); + if (result->error()) { + log().warn("Error while fetching addon tile {}: {}", baseTile->id().toString(), *result->error()); return {}; } - if (auxTile->layerInfo()->type_ != LayerType::Features) { + if (result->layerInfo()->type_ != LayerType::Features) { log().warn("Addon tile is not a feature layer"); return {}; } - return std::static_pointer_cast(auxTile); + return std::static_pointer_cast(result); }(); if (!auxTile) { @@ -578,7 +620,7 @@ struct Service::Impl : public Service::Controller }; Service::Service(Cache::Ptr cache, bool useDataSourceConfig, std::optional defaultTtl) - : impl_(std::make_unique(std::move(cache), useDataSourceConfig, std::move(defaultTtl))) + : impl_(std::make_unique(std::move(cache), useDataSourceConfig, defaultTtl)) { } From ceeb4aa3809bad4fdbe8c2a96b3623e47c63358d Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 29 Jan 2026 18:17:18 +0100 Subject: [PATCH 15/38] Ensure that nextJob does not lock until a to-be-aborted request has been fully served from cache. --- libs/service/include/mapget/service/service.h | 2 +- libs/service/src/service.cpp | 32 ++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/libs/service/include/mapget/service/service.h b/libs/service/include/mapget/service/service.h index c772dd33..edebab07 100644 --- a/libs/service/include/mapget/service/service.h +++ b/libs/service/include/mapget/service/service.h @@ -111,7 +111,7 @@ class LayerTilesRequest // Mutex/condition variable for reading/setting request status. std::mutex statusMutex_; std::condition_variable statusConditionVariable_; - RequestStatus status_ = RequestStatus::Open; + std::atomic status_ = RequestStatus::Open; }; /** diff --git a/libs/service/src/service.cpp b/libs/service/src/service.cpp index d645691f..c8490b19 100644 --- a/libs/service/src/service.cpp +++ b/libs/service/src/service.cpp @@ -40,6 +40,10 @@ LayerTilesRequest::LayerTilesRequest( } void LayerTilesRequest::notifyResult(TileLayer::Ptr r) { + if (isDone()) { + return; + } + const auto type = r->layerInfo()->type_; switch (type) { case LayerType::Features: @@ -70,10 +74,7 @@ void LayerTilesRequest::notifyLoadState(MapTileKey const& key, TileLayer::LoadSt void LayerTilesRequest::setStatus(RequestStatus s) { - { - std::unique_lock statusLock(statusMutex_); - this->status_ = s; - } + this->status_ = s; notifyStatus(); } @@ -162,13 +163,15 @@ struct Service::Controller auto layerIt = i.layers_.find(request->layerId_); // Does the Datasource Info (i) of the worker fit the request? - if (request->mapId_ != i.mapId_ || layerIt == i.layers_.end()) + // Or is it done (/aborted) but not yet removed from requests? + if (request->mapId_ != i.mapId_ || layerIt == i.layers_.end() || request->isDone()) continue; // Find the next pending tile in the request's ordered list. TileId tileId{}; bool foundTile = false; while (request->nextTileIndex_ < request->tiles_.size()) { + // Skip over tiles which were meanwhile done by other workers. tileId = request->tiles_[request->nextTileIndex_++]; if (request->tileIdsNotDone_.find(tileId) != request->tileIdsNotDone_.end()) { foundTile = true; @@ -222,11 +225,17 @@ struct Service::Controller request->notifyLoadState(result->tileKey, TileLayer::LoadState::LoadingQueued); // Move this request to the end of the list, so others gain priority. + // It is ok to manipulate the list here, because we call `break` after the next line. requests_.splice(requests_.end(), requests_, reqIt); log().debug("Working on tile: {}", result->tileKey.toString()); break; } + + // Once unlock and re-lock before we make another sweep over the request list, + // so that it can be updated externally; clients might want to add/remove requests. + jobsMutex_.unlock(); + jobsMutex_.lock(); } while (cachedTilesServedOrInProgressSkipped && !result && anyTasksRemaining); @@ -506,12 +515,13 @@ struct Service::Impl : public Service::Controller void abortRequest(LayerTilesRequest::Ptr const& r) { - std::unique_lock lock(jobsMutex_); - // Remove the request from the list of requests. - auto numRemoved = requests_.remove_if([r](auto&& request) { return r == request; }); - // Clear its jobs to mark it as done. - if (numRemoved) { - r->setStatus(RequestStatus::Aborted); + // Mark the request as aborted. + r->setStatus(RequestStatus::Aborted); + + // Remove the request from the list of requests (needs lock). + { + std::unique_lock lock(jobsMutex_); + requests_.remove_if([r](auto&& request) { return r == request; }); } } From df7b042fe15436fe71978f01f523badecec22955 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 2 Feb 2026 19:30:46 +0100 Subject: [PATCH 16/38] Use more human-readable stat key names. --- libs/model/src/featurelayer.cpp | 2 +- libs/service/include/mapget/service/service.h | 4 +- libs/service/src/datasource.cpp | 2 +- libs/service/src/service.cpp | 39 ++++++++++--------- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/libs/model/src/featurelayer.cpp b/libs/model/src/featurelayer.cpp index cf891b66..e5cb6a56 100644 --- a/libs/model/src/featurelayer.cpp +++ b/libs/model/src/featurelayer.cpp @@ -336,7 +336,7 @@ simfil::model_ptr TileFeatureLayer::newFeature( // contains only references to feature nodes, in the order // of the feature node column. addRoot(simfil::ModelNode::Ptr(result)); - setInfo("num-features", numRoots()); + setInfo("Size/Features", numRoots()); return result; } diff --git a/libs/service/include/mapget/service/service.h b/libs/service/include/mapget/service/service.h index edebab07..cf1cb085 100644 --- a/libs/service/include/mapget/service/service.h +++ b/libs/service/include/mapget/service/service.h @@ -85,7 +85,7 @@ class LayerTilesRequest protected: virtual void notifyResult(TileLayer::Ptr); - void notifyLoadState(MapTileKey const& key, TileLayer::LoadState state); + void notifyLoadState(MapTileKey const& key, TileLayer::LoadState state) const; void setStatus(RequestStatus s); void notifyStatus(); nlohmann::json toJson(); @@ -103,7 +103,7 @@ class LayerTilesRequest size_t nextTileIndex_ = 0; // Track which tiles still need to be scheduled/served for this request. - std::set tileIdsNotDone_; + std::set tileIdsNotStarted_; // So the requester can track how many results have been received. size_t resultCount_ = 0; diff --git a/libs/service/src/datasource.cpp b/libs/service/src/datasource.cpp index 3e7715fe..319c77c0 100644 --- a/libs/service/src/datasource.cpp +++ b/libs/service/src/datasource.cpp @@ -59,7 +59,7 @@ TileLayer::Ptr DataSource::get( // Notify the tile how long it took to fill. if (result) { auto duration = std::chrono::steady_clock::now() - start; - result->setInfo("fill-time-ms", std::chrono::duration_cast(duration).count()); + result->setInfo("Load+Convert/Total#ms", std::chrono::duration_cast(duration).count()); } return result; } diff --git a/libs/service/src/service.cpp b/libs/service/src/service.cpp index c8490b19..aa066c65 100644 --- a/libs/service/src/service.cpp +++ b/libs/service/src/service.cpp @@ -31,7 +31,7 @@ LayerTilesRequest::LayerTilesRequest( layerId_(std::move(layerId)), tiles_(std::move(tiles)) { - tileIdsNotDone_.insert(tiles_.begin(), tiles_.end()); + tileIdsNotStarted_.insert(tiles_.begin(), tiles_.end()); if (tiles_.empty()) { // An empty request is always set to success, but the client/service // is responsible for triggering notifyStatus() in that case. @@ -65,8 +65,7 @@ void LayerTilesRequest::notifyResult(TileLayer::Ptr r) { } } -void LayerTilesRequest::notifyLoadState(MapTileKey const& key, TileLayer::LoadState state) -{ +void LayerTilesRequest::notifyLoadState(MapTileKey const& key, TileLayer::LoadState state) const { if (onLoadStateChanged_) { onLoadStateChanged_(key, state); } @@ -127,6 +126,7 @@ struct Service::Controller MapTileKey tileKey; std::vector waitingRequests; std::optional cacheExpiredAt; + TileLayer::LoadState loadStatus = TileLayer::LoadState::LoadingQueued; }; std::map> jobsInProgress_; // Jobs currently in progress + interested requests @@ -144,11 +144,12 @@ struct Service::Controller raise("Cache must not be null!"); } - std::shared_ptr nextJob(DataSourceInfo const& i) + std::shared_ptr nextJob(DataSourceInfo const& i, std::unique_lock& lock) { // Workers call the nextJob function when they are free. // Note: For thread safety, jobsMutex_ must be held - // when calling this function. + // when calling this function. The lock may be released/re-acquired + // between sweeps to allow external updates. std::shared_ptr result; @@ -173,7 +174,7 @@ struct Service::Controller while (request->nextTileIndex_ < request->tiles_.size()) { // Skip over tiles which were meanwhile done by other workers. tileId = request->tiles_[request->nextTileIndex_++]; - if (request->tileIdsNotDone_.find(tileId) != request->tileIdsNotDone_.end()) { + if (request->tileIdsNotStarted_.find(tileId) != request->tileIdsNotStarted_.end()) { foundTile = true; break; } @@ -186,7 +187,7 @@ struct Service::Controller // Cache lookup. auto cachedResult = cache_->getTileLayer(resultTileKey, i); if (cachedResult.tile) { - request->tileIdsNotDone_.erase(tileId); + request->tileIdsNotStarted_.erase(tileId); log().debug("Serving cached tile: {}", resultTileKey.toString()); request->notifyResult(cachedResult.tile); cachedTilesServedOrInProgressSkipped = true; @@ -200,7 +201,8 @@ struct Service::Controller // can satisfy multiple requests, and allow this request to advance. log().debug("Joining tile with job in progress: {}", resultTileKey.toString()); - request->tileIdsNotDone_.erase(tileId); + request->tileIdsNotStarted_.erase(tileId); + request->notifyLoadState(resultTileKey, inProgress->second->loadStatus); inProgress->second->waitingRequests.push_back(request); cachedTilesServedOrInProgressSkipped = true; continue; @@ -208,7 +210,7 @@ struct Service::Controller // We found something to work on that is not cached and not in progress - // enter it into the jobs-in-progress map with the requesting client. - request->tileIdsNotDone_.erase(tileId); + request->tileIdsNotStarted_.erase(tileId); result = std::make_shared(Job{resultTileKey, {request}, cachedResult.expiredAt}); // Proactively attach other requests that need this tile. for (auto const& otherRequest : requests_) { @@ -216,13 +218,11 @@ struct Service::Controller continue; if (otherRequest->mapId_ != request->mapId_ || otherRequest->layerId_ != request->layerId_) continue; - if (otherRequest->tileIdsNotDone_.erase(tileId) == 0) + if (otherRequest->tileIdsNotStarted_.erase(tileId) == 0) continue; result->waitingRequests.push_back(otherRequest); - otherRequest->notifyLoadState(result->tileKey, TileLayer::LoadState::LoadingQueued); } jobsInProgress_.emplace(result->tileKey, result); - request->notifyLoadState(result->tileKey, TileLayer::LoadState::LoadingQueued); // Move this request to the end of the list, so others gain priority. // It is ok to manipulate the list here, because we call `break` after the next line. @@ -232,15 +232,17 @@ struct Service::Controller break; } - // Once unlock and re-lock before we make another sweep over the request list, - // so that it can be updated externally; clients might want to add/remove requests. - jobsMutex_.unlock(); - jobsMutex_.lock(); + if (cachedTilesServedOrInProgressSkipped && !result && anyTasksRemaining) { + // Unlock and re-lock before we make another sweep over the request list, + // so that it can be updated externally; clients might want to add/remove requests. + lock.unlock(); + lock.lock(); + } } while (cachedTilesServedOrInProgressSkipped && !result && anyTasksRemaining); // Clean up done requests. - requests_.remove_if([](auto&& r) {return r->tileIdsNotDone_.empty(); }); + requests_.remove_if([](auto&& r) {return r->tileIdsNotStarted_.empty(); }); return result; } @@ -286,7 +288,7 @@ struct Service::Worker // is removed. All worker instances are expected to terminate. return true; } - nextJob = controller_.nextJob(info_); + nextJob = controller_.nextJob(info_, lock); return !!nextJob; }); } @@ -306,6 +308,7 @@ struct Service::Worker std::vector waiting; { std::unique_lock lock(controller_.jobsMutex_); + job.loadStatus = state; waiting = job.waitingRequests; } for (auto const& req : waiting) { From 4b6ecfdefb723082e0e3de92c14ebb4dee58f85f Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Tue, 3 Feb 2026 19:41:35 +0100 Subject: [PATCH 17/38] Ensure that libuuid is installed --- .github/workflows/ci-deploy.yaml | 3 +++ .github/workflows/coverage.yml | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-deploy.yaml b/.github/workflows/ci-deploy.yaml index 025d1efa..4d500644 100644 --- a/.github/workflows/ci-deploy.yaml +++ b/.github/workflows/ci-deploy.yaml @@ -50,6 +50,9 @@ jobs: submodules: recursive - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.9 + - name: Install libuuid + run: | + yum -y install libuuid-devel - name: Configure run: | python3 -m venv venv && . ./venv/bin/activate diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 11f22b05..85564cf6 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -20,7 +20,8 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.9 - name: Install dependencies run: | - sudo apt-get install ninja-build + sudo apt-get update + sudo apt-get install -y ninja-build uuid-dev pip install gcovr gcovr --version - name: Configure From bee9fe3c67a111646b5f935a0a9f0114c6190db6 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Tue, 3 Feb 2026 19:42:02 +0100 Subject: [PATCH 18/38] Dedupe requested tiles. --- libs/service/src/service.cpp | 12 ++++++++++-- test/unit/test-http-datasource.cpp | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/libs/service/src/service.cpp b/libs/service/src/service.cpp index aa066c65..7d58d206 100644 --- a/libs/service/src/service.cpp +++ b/libs/service/src/service.cpp @@ -31,8 +31,16 @@ LayerTilesRequest::LayerTilesRequest( layerId_(std::move(layerId)), tiles_(std::move(tiles)) { - tileIdsNotStarted_.insert(tiles_.begin(), tiles_.end()); - if (tiles_.empty()) { + if (!tiles_.empty()) { + std::vector uniqueTiles; + uniqueTiles.reserve(tiles_.size()); + for (const auto& tileId : tiles_) { + if (tileIdsNotStarted_.insert(tileId).second) { + uniqueTiles.push_back(tileId); + } + } + tiles_.swap(uniqueTiles); + } else { // An empty request is always set to success, but the client/service // is responsible for triggering notifyStatus() in that case. status_ = RequestStatus::Success; diff --git a/test/unit/test-http-datasource.cpp b/test/unit/test-http-datasource.cpp index a2f12e2a..5efabdbf 100644 --- a/test/unit/test-http-datasource.cpp +++ b/test/unit/test-http-datasource.cpp @@ -312,9 +312,9 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") client, "Tropico", "WayLayer", - std::vector{{1234, 5678, 9112, 1234}}); + std::vector{{1234, 5678, 9112}}); - REQUIRE(receivedTileCount == 4); + REQUIRE(receivedTileCount == 3); REQUIRE(request->getStatus() == RequestStatus::Success); } From 35990add5d79c6c9bf0d55bf3b960fbe047daef6 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Wed, 4 Feb 2026 09:58:08 +0100 Subject: [PATCH 19/38] Avoid acccidental call to Tile(x,y,z) constructor. --- test/unit/test-http-datasource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test-http-datasource.cpp b/test/unit/test-http-datasource.cpp index 5efabdbf..1f6abecb 100644 --- a/test/unit/test-http-datasource.cpp +++ b/test/unit/test-http-datasource.cpp @@ -312,7 +312,7 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") client, "Tropico", "WayLayer", - std::vector{{1234, 5678, 9112}}); + std::vector{1234, 5678, 9112}); REQUIRE(receivedTileCount == 3); REQUIRE(request->getStatus() == RequestStatus::Success); From efb1b5380b47d22b3ac9a92705fe27a28cf18d72 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Wed, 4 Feb 2026 11:01:27 +0100 Subject: [PATCH 20/38] Fix sonar complaints. --- libs/http-service/src/tiles-ws-controller.cpp | 21 +- test/unit/test-http-datasource.cpp | 436 ++++++++---------- 2 files changed, 217 insertions(+), 240 deletions(-) diff --git a/libs/http-service/src/tiles-ws-controller.cpp b/libs/http-service/src/tiles-ws-controller.cpp index 6d7f5d23..0651d3b1 100644 --- a/libs/http-service/src/tiles-ws-controller.cpp +++ b/libs/http-service/src/tiles-ws-controller.cpp @@ -168,12 +168,12 @@ class TilesWsSession : public std::enable_shared_from_this auto& req = requests_[i]; req->onFeatureLayer([weak](auto&& layer) { if (auto self = weak.lock()) { - self->onTileLayer(std::move(layer)); + self->onTileLayer(std::forward(layer)); } }); req->onSourceDataLayer([weak](auto&& layer) { if (auto self = weak.lock()) { - self->onTileLayer(std::move(layer)); + self->onTileLayer(std::forward(layer)); } }); req->onLayerLoadStateChanged([weak](MapTileKey const& key, TileLayer::LoadState state) { @@ -264,20 +264,19 @@ class TilesWsSession : public std::enable_shared_from_this void onWriterMessage(std::string msg, TileLayerStream::MessageType type) { // Writer messages are only generated from within onTileLayer under mutex_. - if (!currentWriteBatch_) { + if (!currentWriteBatch_.has_value()) { raise("TilesWsSession writer callback used out-of-band"); } currentWriteBatch_->push_back(WriterMessage{std::move(msg), type}); } - void onTileLayer(TileLayer::Ptr layer) + void onTileLayer(TileLayer::Ptr const& layer) { if (cancelled_) return; if (!layer) return; - std::vector batch; std::optional> stringPoolCommit; { @@ -285,9 +284,13 @@ class TilesWsSession : public std::enable_shared_from_this if (cancelled_) return; - currentWriteBatch_ = &batch; + if (currentWriteBatch_.has_value()) { + raise("TilesWsSession writer callback re-entered"); + } + currentWriteBatch_.emplace(); writer_->write(layer); - currentWriteBatch_ = nullptr; + auto batch = std::move(*currentWriteBatch_); + currentWriteBatch_.reset(); // If a StringPool message was generated, the writer updates offsets_ // to the new highest string ID for this node after emitting it. @@ -400,7 +403,7 @@ class TilesWsSession : public std::enable_shared_from_this }).dump(); } - [[nodiscard]] std::string buildLoadStatePayload(MapTileKey const& key, TileLayer::LoadState state) + static std::string buildLoadStatePayload(MapTileKey const& key, TileLayer::LoadState state) { return nlohmann::json::object({ {"type", "mapget.tiles.load-state"}, @@ -479,7 +482,7 @@ class TilesWsSession : public std::enable_shared_from_this TileLayerStream::StringPoolOffsetMap offsets_; std::unique_ptr writer_; - std::vector* currentWriteBatch_ = nullptr; + std::optional> currentWriteBatch_; std::atomic_bool drainScheduled_{false}; std::atomic_bool cancelled_{false}; diff --git a/test/unit/test-http-datasource.cpp b/test/unit/test-http-datasource.cpp index 1f6abecb..30cdccd3 100644 --- a/test/unit/test-http-datasource.cpp +++ b/test/unit/test-http-datasource.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -82,6 +83,165 @@ class SyncHttpClient drogon::HttpClientPtr client_; }; +class WsTilesClient +{ +public: + WsTilesClient(uint16_t port, std::shared_ptr layerInfo, bool requireFeatureLayer = true) + : layerInfo_(std::move(layerInfo)), + requireFeatureLayer_(requireFeatureLayer), + reader_( + [this](auto&&, auto&&) { return layerInfo_; }, + [this](auto&& tile) { + if (requireFeatureLayer_ && tile->id().layer_ != LayerType::Features) { + setError("Unexpected tile layer type"); + return; + } + receivedTileCount_.fetch_add(1, std::memory_order_relaxed); + }) + { + loopThread_ = std::make_unique("MapgetTestWsClient"); + loopThread_->run(); + + client_ = drogon::WebSocketClient::newWebSocketClient( + fmt::format("ws://127.0.0.1:{}", port), + loopThread_->getLoop()); + + client_->setMessageHandler( + [this](std::string&& msg, + const drogon::WebSocketClientPtr&, + const drogon::WebSocketMessageType& msgType) { + if (msgType != drogon::WebSocketMessageType::Binary) { + return; + } + handleBinaryMessage(std::move(msg)); + }); + } + + bool connect(bool sendAuthHeader) + { + auto connectReq = drogon::HttpRequest::newHttpRequest(); + connectReq->setMethod(drogon::Get); + connectReq->setPath("/tiles"); + if (sendAuthHeader) { + connectReq->addHeader("X-USER-ROLE", "Tropico-Viewer"); + } + + std::promise connectPromise; + auto connectFuture = connectPromise.get_future(); + client_->connectToServer( + connectReq, + [&connectPromise]( + drogon::ReqResult result, + const drogon::HttpResponsePtr&, + const drogon::WebSocketClientPtr&) { connectPromise.set_value(result); }); + + if (connectFuture.wait_for(std::chrono::seconds(5)) != std::future_status::ready) { + return false; + } + return connectFuture.get() == drogon::ReqResult::Ok; + } + + drogon::WebSocketConnectionPtr connection() const { return client_->getConnection(); } + + void send(std::string_view payload) + { + auto conn = connection(); + if (conn && conn->connected()) { + conn->send(std::string(payload), drogon::WebSocketMessageType::Text); + } + } + + [[nodiscard]] bool waitForDone(std::chrono::seconds timeout) + { + std::unique_lock lock(mutex_); + return cv_.wait_for(lock, timeout, [this] { + return !error_.empty() || (lastStatus_.has_value() && lastStatus_->value("allDone", false)); + }); + } + + void resetStatus() + { + std::lock_guard lock(mutex_); + lastStatus_.reset(); + error_.clear(); + } + + void resetTileCount() { receivedTileCount_.store(0, std::memory_order_relaxed); } + + std::optional lastStatus() const + { + std::lock_guard lock(mutex_); + return lastStatus_; + } + + std::string error() const + { + std::lock_guard lock(mutex_); + return error_; + } + + int receivedTileCount() const { return receivedTileCount_.load(std::memory_order_relaxed); } + + void stop() { client_->stop(); } + +private: + void handleBinaryMessage(std::string&& msg) + { + TileLayerStream::MessageType type = TileLayerStream::MessageType::None; + uint32_t payloadSize = 0; + std::stringstream ss; + ss.write(msg.data(), static_cast(msg.size())); + if (!TileLayerStream::Reader::readMessageHeader(ss, type, payloadSize)) { + setError("Failed to read stream message header"); + return; + } + + if (type == TileLayerStream::MessageType::Status) { + std::string payload(payloadSize, '\0'); + ss.read(payload.data(), static_cast(payloadSize)); + try { + auto parsed = nlohmann::json::parse(payload); + { + std::lock_guard lock(mutex_); + lastStatus_ = std::move(parsed); + } + cv_.notify_all(); + } + catch (const std::exception& e) { + setError(std::string("Failed to parse status JSON: ") + e.what()); + } + return; + } + + try { + reader_.read(msg); + } + catch (const std::exception& e) { + setError(std::string("Failed to parse tile stream: ") + e.what()); + } + } + + void setError(std::string message) + { + { + std::lock_guard lock(mutex_); + error_ = std::move(message); + } + cv_.notify_all(); + } + + mutable std::mutex mutex_; + std::condition_variable cv_; + std::optional lastStatus_; + std::string error_; + std::atomic_int receivedTileCount_{0}; + std::unique_ptr loopThread_; + drogon::WebSocketClientPtr client_; + std::shared_ptr layerInfo_; + bool requireFeatureLayer_{true}; + TileLayerStream::Reader reader_; +}; + class ChildProcessWithPort { public: @@ -390,127 +550,36 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") REQUIRE(receivedTileCount == 1); } - auto runWsTilesRequest = [&](bool sendAuthHeader, std::string requestJson) { - auto wsLoopThread = std::make_unique("MapgetTestWsClient"); - wsLoopThread->run(); - - auto wsClient = drogon::WebSocketClient::newWebSocketClient( - fmt::format("ws://127.0.0.1:{}", service.port()), - wsLoopThread->getLoop()); - - std::mutex mutex; - std::condition_variable cv; - std::optional lastStatus; - std::atomic_int receivedTileCount{0}; - std::string error; - - const auto dsInfo = remoteDataSource->info(); - const auto layerInfo = dsInfo.getLayer("WayLayer"); - REQUIRE(layerInfo != nullptr); - - TileLayerStream::Reader reader( - [&](auto&&, auto&&) { return layerInfo; }, - [&](auto&& tile) { - if (tile->id().layer_ != LayerType::Features) { - std::lock_guard lock(mutex); - error = "Unexpected tile layer type"; - } - receivedTileCount.fetch_add(1, std::memory_order_relaxed); - }); - - wsClient->setMessageHandler( - [&](std::string&& msg, - const drogon::WebSocketClientPtr&, - const drogon::WebSocketMessageType& msgType) { - if (msgType != drogon::WebSocketMessageType::Binary) { - return; - } - - TileLayerStream::MessageType type = TileLayerStream::MessageType::None; - uint32_t payloadSize = 0; - std::stringstream ss; - ss.write(msg.data(), static_cast(msg.size())); - if (!TileLayerStream::Reader::readMessageHeader(ss, type, payloadSize)) { - std::lock_guard lock(mutex); - error = "Failed to read stream message header"; - cv.notify_all(); - return; - } - - if (type == TileLayerStream::MessageType::Status) { - std::string payload(payloadSize, '\0'); - ss.read(payload.data(), static_cast(payloadSize)); - nlohmann::json parsed; - try { - parsed = nlohmann::json::parse(payload); - } - catch (const std::exception& e) { - std::lock_guard lock(mutex); - error = std::string("Failed to parse status JSON: ") + e.what(); - cv.notify_all(); - return; - } - { - std::lock_guard lock(mutex); - lastStatus = std::move(parsed); - } - cv.notify_all(); - return; - } - - try { - reader.read(msg); - } - catch (const std::exception& e) { - std::lock_guard lock(mutex); - error = std::string("Failed to parse tile stream: ") + e.what(); - cv.notify_all(); - } - }); - - auto connectReq = drogon::HttpRequest::newHttpRequest(); - connectReq->setMethod(drogon::Get); - connectReq->setPath("/tiles"); - if (sendAuthHeader) { - connectReq->addHeader("X-USER-ROLE", "Tropico-Viewer"); - } - - std::promise connectPromise; - auto connectFuture = connectPromise.get_future(); - wsClient->connectToServer( - connectReq, - [&connectPromise]( - drogon::ReqResult result, - const drogon::HttpResponsePtr&, - const drogon::WebSocketClientPtr&) { connectPromise.set_value(result); }); - - REQUIRE(connectFuture.wait_for(std::chrono::seconds(5)) == std::future_status::ready); - REQUIRE(connectFuture.get() == drogon::ReqResult::Ok); + const auto dsInfo = remoteDataSource->info(); + const auto layerInfo = dsInfo.getLayer("WayLayer"); + REQUIRE(layerInfo != nullptr); - auto conn = wsClient->getConnection(); + auto requireConnected = [](WsTilesClient& wsClient) { + auto conn = wsClient.connection(); if (!conn || !conn->connected()) { - wsClient->stop(); + wsClient.stop(); FAIL("WebSocket connection not established"); } + return conn; + }; - conn->send(requestJson, drogon::WebSocketMessageType::Text); + auto runWsTilesRequest = [&](bool sendAuthHeader, const std::string& requestJson) { + WsTilesClient wsClient(service.port(), layerInfo); - { - std::unique_lock lock(mutex); - REQUIRE(cv.wait_for(lock, std::chrono::seconds(10), [&] { - return !error.empty() || - (lastStatus.has_value() && lastStatus->value("allDone", false)); - })); - if (!error.empty()) { - wsClient->stop(); - FAIL(error); - } + REQUIRE(wsClient.connect(sendAuthHeader)); + requireConnected(wsClient)->send(requestJson, drogon::WebSocketMessageType::Text); + + REQUIRE(wsClient.waitForDone(std::chrono::seconds(10))); + if (!wsClient.error().empty()) { + wsClient.stop(); + FAIL(wsClient.error()); } - wsClient->stop(); + auto status = wsClient.lastStatus(); + wsClient.stop(); - REQUIRE(lastStatus.has_value()); - return std::make_tuple(*lastStatus, receivedTileCount.load(std::memory_order_relaxed)); + REQUIRE(status.has_value()); + return std::make_tuple(*status, wsClient.receivedTileCount()); }; // WebSocket tiles: unauthorized without auth header. @@ -532,123 +601,30 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") // WebSocket tiles: invalid request stays on the same connection, then succeeds. { - auto wsLoopThread = std::make_unique("MapgetTestWsClientReuse"); - wsLoopThread->run(); - - auto wsClient = drogon::WebSocketClient::newWebSocketClient( - fmt::format("ws://127.0.0.1:{}", service.port()), - wsLoopThread->getLoop()); - - std::mutex mutex; - std::condition_variable cv; - std::optional lastStatus; - std::atomic_int receivedTileCount{0}; - std::string error; - - const auto dsInfo = remoteDataSource->info(); - const auto layerInfo = dsInfo.getLayer("WayLayer"); - REQUIRE(layerInfo != nullptr); - - TileLayerStream::Reader reader( - [&](auto&&, auto&&) { return layerInfo; }, - [&](auto&&) { receivedTileCount.fetch_add(1, std::memory_order_relaxed); }); - - wsClient->setMessageHandler( - [&](std::string&& msg, - const drogon::WebSocketClientPtr&, - const drogon::WebSocketMessageType& msgType) { - if (msgType != drogon::WebSocketMessageType::Binary) { - return; - } - - TileLayerStream::MessageType type = TileLayerStream::MessageType::None; - uint32_t payloadSize = 0; - std::stringstream ss; - ss.write(msg.data(), static_cast(msg.size())); - if (!TileLayerStream::Reader::readMessageHeader(ss, type, payloadSize)) { - std::lock_guard lock(mutex); - error = "Failed to read stream message header"; - cv.notify_all(); - return; - } - - if (type == TileLayerStream::MessageType::Status) { - std::string payload(payloadSize, '\0'); - ss.read(payload.data(), static_cast(payloadSize)); - nlohmann::json parsed; - try { - parsed = nlohmann::json::parse(payload); - } - catch (const std::exception& e) { - std::lock_guard lock(mutex); - error = std::string("Failed to parse status JSON: ") + e.what(); - cv.notify_all(); - return; - } - { - std::lock_guard lock(mutex); - lastStatus = std::move(parsed); - } - cv.notify_all(); - return; - } - - try { - reader.read(msg); - } - catch (const std::exception& e) { - std::lock_guard lock(mutex); - error = std::string("Failed to parse tile stream: ") + e.what(); - cv.notify_all(); - } - }); - - auto connectReq = drogon::HttpRequest::newHttpRequest(); - connectReq->setMethod(drogon::Get); - connectReq->setPath("/tiles"); - connectReq->addHeader("X-USER-ROLE", "Tropico-Viewer"); - - std::promise connectPromise; - auto connectFuture = connectPromise.get_future(); - wsClient->connectToServer( - connectReq, - [&connectPromise]( - drogon::ReqResult result, - const drogon::HttpResponsePtr&, - const drogon::WebSocketClientPtr&) { connectPromise.set_value(result); }); - - REQUIRE(connectFuture.wait_for(std::chrono::seconds(5)) == std::future_status::ready); - REQUIRE(connectFuture.get() == drogon::ReqResult::Ok); - - auto conn = wsClient->getConnection(); - if (!conn || !conn->connected()) { - wsClient->stop(); - FAIL("WebSocket connection not established"); - } + WsTilesClient wsClient(service.port(), layerInfo); + REQUIRE(wsClient.connect(true)); + + auto conn = requireConnected(wsClient); // Invalid JSON: should yield a Status message but keep the socket open. { conn->send("{not json", drogon::WebSocketMessageType::Text); - std::unique_lock lock(mutex); - REQUIRE(cv.wait_for(lock, std::chrono::seconds(5), [&] { - return !error.empty() || - (lastStatus.has_value() && lastStatus->value("allDone", false)); - })); - if (!error.empty()) { - wsClient->stop(); - FAIL(error); + REQUIRE(wsClient.waitForDone(std::chrono::seconds(5))); + if (!wsClient.error().empty()) { + wsClient.stop(); + FAIL(wsClient.error()); } - REQUIRE(lastStatus->value("message", "").find("Invalid JSON") != std::string::npos); + + auto status = wsClient.lastStatus(); + REQUIRE(status.has_value()); + REQUIRE(status->value("message", "").find("Invalid JSON") != std::string::npos); REQUIRE(conn->connected()); } // Valid request should succeed afterwards. { - { - std::lock_guard lock(mutex); - lastStatus.reset(); - } - receivedTileCount.store(0, std::memory_order_relaxed); + wsClient.resetStatus(); + wsClient.resetTileCount(); auto req = nlohmann::json::object({ {"requests", nlohmann::json::array({nlohmann::json::object({ @@ -660,20 +636,18 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") conn->send(req, drogon::WebSocketMessageType::Text); - std::unique_lock lock(mutex); - REQUIRE(cv.wait_for(lock, std::chrono::seconds(10), [&] { - return !error.empty() || - (lastStatus.has_value() && lastStatus->value("allDone", false)); - })); - if (!error.empty()) { - wsClient->stop(); - FAIL(error); + REQUIRE(wsClient.waitForDone(std::chrono::seconds(10))); + if (!wsClient.error().empty()) { + wsClient.stop(); + FAIL(wsClient.error()); } - REQUIRE(receivedTileCount.load(std::memory_order_relaxed) == 1); - REQUIRE(lastStatus->contains("requests")); - REQUIRE((*lastStatus)["requests"].size() == 1); - REQUIRE((*lastStatus)["requests"][0]["status"].get() == + auto status = wsClient.lastStatus(); + REQUIRE(wsClient.receivedTileCount() == 1); + REQUIRE(status.has_value()); + REQUIRE(status->contains("requests")); + REQUIRE((*status)["requests"].size() == 1); + REQUIRE((*status)["requests"][0]["status"].get() == static_cast(RequestStatus::Success)); } From b2dc3a89270518a734ed21a194e780486c535292 Mon Sep 17 00:00:00 2001 From: Wagram Airiian Date: Wed, 4 Feb 2026 09:43:51 +0100 Subject: [PATCH 21/38] Update reference --- cmake/deps.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/deps.cmake b/cmake/deps.cmake index 323dc283..89300df1 100644 --- a/cmake/deps.cmake +++ b/cmake/deps.cmake @@ -15,7 +15,7 @@ CPMAddPackage( "EXPECTED_BUILD_TESTS OFF" "EXPECTED_BUILD_PACKAGE_DEB OFF") CPMAddPackage( - URI "gh:Klebert-Engineering/simfil@0.6.3#v0.6.3" + URI "gh:Klebert-Engineering/simfil#byte-array" OPTIONS "SIMFIL_WITH_MODEL_JSON ON" "SIMFIL_SHARED OFF") From 844ac78cbea69e688b53ace79e346d8fe9c1c358 Mon Sep 17 00:00:00 2001 From: Wagram Airiian Date: Wed, 4 Feb 2026 09:52:34 +0100 Subject: [PATCH 22/38] Add byte array --- libs/model/include/mapget/model/featureid.h | 2 ++ libs/model/src/featureid.cpp | 19 ++++++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/libs/model/include/mapget/model/featureid.h b/libs/model/include/mapget/model/featureid.h index e9c18b0e..8a6a35ff 100644 --- a/libs/model/include/mapget/model/featureid.h +++ b/libs/model/include/mapget/model/featureid.h @@ -67,6 +67,8 @@ class FeatureId : public simfil::MandatoryDerivedModelNodeBase Data* data_ = nullptr; model_ptr fields_; + + mutable std::vector byteArrayCache_; }; } diff --git a/libs/model/src/featureid.cpp b/libs/model/src/featureid.cpp index ebd3f877..92d19626 100644 --- a/libs/model/src/featureid.cpp +++ b/libs/model/src/featureid.cpp @@ -27,8 +27,11 @@ std::string FeatureId::toString() const auto addIdPart = [&result](auto&& v) { - if constexpr (!std::is_same_v, std::monostate>) + if constexpr (std::is_same_v, simfil::ByteArray>) { + result << "." << v.toDisplayString(); + } else if constexpr (!std::is_same_v, std::monostate>) { result << "." << v; + } }; // Add common id-part fields @@ -84,14 +87,24 @@ bool FeatureId::iterate(const simfil::ModelNode::IterCallback& cb) const KeyValueViewPairs FeatureId::keyValuePairs() const { KeyValueViewPairs result; + byteArrayCache_.clear(); auto objectFieldsToKeyValuePairs = [&result, this](simfil::ModelNode::FieldRange fields){ for (auto const& [key, value] : fields) { auto keyStr = model().strings()->resolve(key); std::visit( - [&result, &keyStr](auto&& v) + [&result, &keyStr, this](auto&& v) { - if constexpr (!std::is_same_v, std::monostate> && !std::is_same_v, double>) { + using T = std::decay_t; + if constexpr (std::is_same_v || std::is_same_v) { + return; + } else if constexpr (std::is_same_v) { + byteArrayCache_.emplace_back(v.toDisplayString()); + result.emplace_back(*keyStr, std::string_view(byteArrayCache_.back())); + } else if constexpr (std::is_same_v) { + byteArrayCache_.push_back(v); + result.emplace_back(*keyStr, std::string_view(byteArrayCache_.back())); + } else { result.emplace_back(*keyStr, v); } }, From 43a8217d891a02a4a388045460d5878cfbfd501c Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Wed, 4 Feb 2026 12:11:47 +0100 Subject: [PATCH 23/38] Bump version. --- CMakeLists.txt | 2 +- test/unit/test-http-datasource.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f97065b2..db6ed39b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(mapget LANGUAGES CXX C) # Allow version to be set from command line for CI/CD # For local development, use the default version if(NOT DEFINED MAPGET_VERSION) - set(MAPGET_VERSION 2025.5.1) + set(MAPGET_VERSION 2026.1.0) endif() set(CMAKE_CXX_STANDARD 20) diff --git a/test/unit/test-http-datasource.cpp b/test/unit/test-http-datasource.cpp index 30cdccd3..0b22852e 100644 --- a/test/unit/test-http-datasource.cpp +++ b/test/unit/test-http-datasource.cpp @@ -651,7 +651,7 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") static_cast(RequestStatus::Success)); } - wsClient->stop(); + wsClient.stop(); } } From 98c62f76b72c26b97409423d19494746dbdcdcf7 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Wed, 4 Feb 2026 12:47:26 +0100 Subject: [PATCH 24/38] Ensure that the cache DB is not busy when running tests. --- .../detect-ports-and-prepare-config-yaml.py | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/test/integration/detect-ports-and-prepare-config-yaml.py b/test/integration/detect-ports-and-prepare-config-yaml.py index 0360ccdd..a1963b29 100644 --- a/test/integration/detect-ports-and-prepare-config-yaml.py +++ b/test/integration/detect-ports-and-prepare-config-yaml.py @@ -45,6 +45,30 @@ def _patch_sample_service_yaml(text: str, mapget_port: int, datasource_cpp_port: return text +def _patch_cache_dir(text: str, cache_path: str) -> str: + # Prefer updating an existing cache-dir value, otherwise insert after cache-type. + escaped_path = cache_path.replace("'", "''") + cache_value = f"'{escaped_path}'" + if re.search(r"(?m)^\s*cache-dir:\s*.*$", text): + return re.sub( + r"(?m)^(\s*cache-dir:\s*).*$", + rf"\g<1>{cache_value}", + text, + count=1, + ) + match = re.search(r"(?m)^(\s*)cache-type:\s*.*$", text) + if not match: + return text + indent = match.group(1) + insert_line = f"{indent}cache-dir: {cache_value}" + return re.sub( + r"(?m)^(\s*cache-type:\s*.*)$", + rf"\g<1>\n{insert_line}", + text, + count=1, + ) + + def _patch_sample_fetch_yaml(text: str, mapget_port: int) -> str: return re.sub( r"(?m)^(\s*server:\s*127\.0\.0\.1:)\d+(\s*)$", @@ -83,8 +107,12 @@ def main() -> int: examples_config = repo_root / "examples" / "config" sample_service = (examples_config / "sample-service.yaml").read_text(encoding="utf-8") + cache_path = str((out_dir / "mapget-cache.db").resolve()) (out_dir / "sample-service.yaml").write_text( - _patch_sample_service_yaml(sample_service, mapget_port, datasource_cpp_port, datasource_py_port), + _patch_cache_dir( + _patch_sample_service_yaml(sample_service, mapget_port, datasource_cpp_port, datasource_py_port), + cache_path, + ), encoding="utf-8", newline="\n", ) @@ -108,4 +136,3 @@ def main() -> int: if __name__ == "__main__": raise SystemExit(main()) - From bdf24c9e74426de474d86e489c4362ab7638edcf Mon Sep 17 00:00:00 2001 From: Wagram Airiian Date: Wed, 4 Feb 2026 14:41:16 +0100 Subject: [PATCH 25/38] Raise exception in FeatureId when value is ByteArray --- libs/model/include/mapget/model/featureid.h | 2 -- libs/model/src/featureid.cpp | 18 +++++++----------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/libs/model/include/mapget/model/featureid.h b/libs/model/include/mapget/model/featureid.h index 8a6a35ff..e9c18b0e 100644 --- a/libs/model/include/mapget/model/featureid.h +++ b/libs/model/include/mapget/model/featureid.h @@ -67,8 +67,6 @@ class FeatureId : public simfil::MandatoryDerivedModelNodeBase Data* data_ = nullptr; model_ptr fields_; - - mutable std::vector byteArrayCache_; }; } diff --git a/libs/model/src/featureid.cpp b/libs/model/src/featureid.cpp index 92d19626..2fbc8b43 100644 --- a/libs/model/src/featureid.cpp +++ b/libs/model/src/featureid.cpp @@ -3,6 +3,8 @@ #include +#include "mapget/log.h" + namespace mapget { @@ -87,24 +89,18 @@ bool FeatureId::iterate(const simfil::ModelNode::IterCallback& cb) const KeyValueViewPairs FeatureId::keyValuePairs() const { KeyValueViewPairs result; - byteArrayCache_.clear(); auto objectFieldsToKeyValuePairs = [&result, this](simfil::ModelNode::FieldRange fields){ for (auto const& [key, value] : fields) { auto keyStr = model().strings()->resolve(key); std::visit( - [&result, &keyStr, this](auto&& v) + [&result, &keyStr](auto&& v) { using T = std::decay_t; - if constexpr (std::is_same_v || std::is_same_v) { - return; - } else if constexpr (std::is_same_v) { - byteArrayCache_.emplace_back(v.toDisplayString()); - result.emplace_back(*keyStr, std::string_view(byteArrayCache_.back())); - } else if constexpr (std::is_same_v) { - byteArrayCache_.push_back(v); - result.emplace_back(*keyStr, std::string_view(byteArrayCache_.back())); - } else { + if constexpr (std::is_same_v) { + raiseFmt("FeatureId part '{}' cannot be a ByteArray.", keyStr ? *keyStr : ""); + } + else if constexpr (!std::is_same_v && !std::is_same_v) { result.emplace_back(*keyStr, v); } }, From 6919cff184c125260f4d65cdcc45b3ebaee462f2 Mon Sep 17 00:00:00 2001 From: Wagram Airiian Date: Wed, 4 Feb 2026 14:44:22 +0100 Subject: [PATCH 26/38] Raise exception in FeatureId when value is ByteArray --- libs/model/src/featureid.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/model/src/featureid.cpp b/libs/model/src/featureid.cpp index 2fbc8b43..cb8293d5 100644 --- a/libs/model/src/featureid.cpp +++ b/libs/model/src/featureid.cpp @@ -30,7 +30,7 @@ std::string FeatureId::toString() const auto addIdPart = [&result](auto&& v) { if constexpr (std::is_same_v, simfil::ByteArray>) { - result << "." << v.toDisplayString(); + raiseFmt("FeatureId part value '{}' cannot be a ByteArray.", v.toDisplayString()); } else if constexpr (!std::is_same_v, std::monostate>) { result << "." << v; } From 8fb27944f17c53abdc7fecae8cb7a0dea7f8305b Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Wed, 4 Feb 2026 12:47:26 +0100 Subject: [PATCH 27/38] Use passkey pattern to make Node construction rights more obvious to Intellisense. --- libs/model/include/mapget/model/attr.h | 12 ++- libs/model/include/mapget/model/attrlayer.h | 22 +++-- libs/model/include/mapget/model/feature.h | 21 +++- libs/model/include/mapget/model/featureid.h | 12 ++- libs/model/include/mapget/model/geometry.h | 57 ++++++----- libs/model/include/mapget/model/pointnode.h | 19 ++-- libs/model/include/mapget/model/relation.h | 12 ++- libs/model/include/mapget/model/sourcedata.h | 17 +++- .../mapget/model/sourcedatareference.h | 27 ++++-- libs/model/include/mapget/model/validity.h | 18 ++-- libs/model/src/attr.cpp | 8 +- libs/model/src/attrlayer.cpp | 10 +- libs/model/src/feature.cpp | 18 +++- libs/model/src/featureid.cpp | 7 +- libs/model/src/featurelayer.cpp | 97 ++++++++++++------- libs/model/src/geometry.cpp | 72 ++++++++------ libs/model/src/pointnode.cpp | 20 ++-- libs/model/src/relation.cpp | 8 +- libs/model/src/sourcedata.cpp | 17 +++- libs/model/src/sourcedatalayer.cpp | 9 +- libs/model/src/sourcedatareference.cpp | 18 +++- libs/model/src/validity.cpp | 6 +- 22 files changed, 338 insertions(+), 169 deletions(-) diff --git a/libs/model/include/mapget/model/attr.h b/libs/model/include/mapget/model/attr.h index 5fef0b17..a4a40a07 100644 --- a/libs/model/include/mapget/model/attr.h +++ b/libs/model/include/mapget/model/attr.h @@ -17,7 +17,6 @@ class Geometry; class Attribute : public simfil::ProceduralObject<2, Attribute, TileFeatureLayer> { friend class TileFeatureLayer; - template friend struct simfil::model_ptr; public: /** @@ -63,12 +62,19 @@ class Attribute : public simfil::ProceduralObject<2, Attribute, TileFeatureLayer } }; - Attribute(Data* data, simfil::ModelConstPtr l, simfil::ModelNodeAddress a); - Attribute() = default; +public: + explicit Attribute(simfil::detail::mp_key key) + : simfil::ProceduralObject<2, Attribute, TileFeatureLayer>(key) {} + Attribute(Data* data, + simfil::ModelConstPtr l, + simfil::ModelNodeAddress a, + simfil::detail::mp_key key); + Attribute() = delete; /** * Pointer to the actual data stored for the attribute. */ +protected: Data* data_ = nullptr; }; diff --git a/libs/model/include/mapget/model/attrlayer.h b/libs/model/include/mapget/model/attrlayer.h index f0225ef0..e51db816 100644 --- a/libs/model/include/mapget/model/attrlayer.h +++ b/libs/model/include/mapget/model/attrlayer.h @@ -18,7 +18,6 @@ class AttributeLayer : public simfil::Object { friend class TileFeatureLayer; friend class bitsery::Access; - template friend struct simfil::model_ptr; public: /** @@ -40,9 +39,13 @@ class AttributeLayer : public simfil::Object */ bool forEachAttribute(std::function const& attr)> const& cb) const; -protected: - AttributeLayer(simfil::ArrayIndex i, simfil::ModelConstPtr l, simfil::ModelNodeAddress a); - AttributeLayer() = default; +public: + explicit AttributeLayer(simfil::detail::mp_key key) : simfil::Object(key) {} + AttributeLayer(simfil::ArrayIndex i, + simfil::ModelConstPtr l, + simfil::ModelNodeAddress a, + simfil::detail::mp_key key); + AttributeLayer() = delete; }; /** @@ -55,7 +58,6 @@ class AttributeLayerList : public simfil::Object friend class TileFeatureLayer; friend class bitsery::Access; friend class Feature; - template friend struct simfil::model_ptr; public: /** @@ -78,9 +80,13 @@ class AttributeLayerList : public simfil::Object std::function const& layer)> const& cb ) const; -protected: - AttributeLayerList(simfil::ArrayIndex i, simfil::ModelConstPtr l, simfil::ModelNodeAddress a); - AttributeLayerList() = default; +public: + explicit AttributeLayerList(simfil::detail::mp_key key) : simfil::Object(key) {} + AttributeLayerList(simfil::ArrayIndex i, + simfil::ModelConstPtr l, + simfil::ModelNodeAddress a, + simfil::detail::mp_key key); + AttributeLayerList() = delete; }; } diff --git a/libs/model/include/mapget/model/feature.h b/libs/model/include/mapget/model/feature.h index 8e5dbc69..66602203 100644 --- a/libs/model/include/mapget/model/feature.h +++ b/libs/model/include/mapget/model/feature.h @@ -55,7 +55,6 @@ class Feature : public simfil::MandatoryDerivedModelNodeBase friend class bitsery::Access; friend class TileFeatureLayer; friend class BoundFeature; - template friend struct simfil::model_ptr; public: /** Get the name of this feature's type. */ @@ -201,9 +200,16 @@ class Feature : public simfil::MandatoryDerivedModelNodeBase } }; - Feature(Data& d, simfil::ModelConstPtr l, simfil::ModelNodeAddress a); - Feature() = default; +public: + explicit Feature(simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(key) {} + Feature(Data& d, + simfil::ModelConstPtr l, + simfil::ModelNodeAddress a, + simfil::detail::mp_key key); + Feature() = delete; +protected: Data* data_ = nullptr; // We keep the fields in a tiny vector on the stack, @@ -221,8 +227,13 @@ class Feature : public simfil::MandatoryDerivedModelNodeBase [[nodiscard]] simfil::StringId keyAt(int64_t) const override; [[nodiscard]] bool iterate(IterCallback const& cb) const override; - FeaturePropertyView(Data& d, simfil::ModelConstPtr l, simfil::ModelNodeAddress a); - FeaturePropertyView() = default; + explicit FeaturePropertyView(simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(key) {} + FeaturePropertyView(Data& d, + simfil::ModelConstPtr l, + simfil::ModelNodeAddress a, + simfil::detail::mp_key key); + FeaturePropertyView() = delete; Data* data_ = nullptr; model_ptr attrs_; diff --git a/libs/model/include/mapget/model/featureid.h b/libs/model/include/mapget/model/featureid.h index e9c18b0e..ab5274dd 100644 --- a/libs/model/include/mapget/model/featureid.h +++ b/libs/model/include/mapget/model/featureid.h @@ -24,7 +24,6 @@ class FeatureId : public simfil::MandatoryDerivedModelNodeBase friend class Feature; friend class Relation; friend class bitsery::Access; - template friend struct simfil::model_ptr; public: /** Convert the FeatureId to a string like `....` */ @@ -61,9 +60,16 @@ class FeatureId : public simfil::MandatoryDerivedModelNodeBase } }; - FeatureId(Data& data, simfil::ModelConstPtr l, simfil::ModelNodeAddress a); - FeatureId() = default; +public: + explicit FeatureId(simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(key) {} + FeatureId(Data& data, + simfil::ModelConstPtr l, + simfil::ModelNodeAddress a, + simfil::detail::mp_key key); + FeatureId() = delete; +protected: Data* data_ = nullptr; model_ptr fields_; diff --git a/libs/model/include/mapget/model/geometry.h b/libs/model/include/mapget/model/geometry.h index c90a43f3..f0de2f29 100644 --- a/libs/model/include/mapget/model/geometry.h +++ b/libs/model/include/mapget/model/geometry.h @@ -45,7 +45,6 @@ struct SelfContainedGeometry class Geometry final : public simfil::MandatoryDerivedModelNodeBase { public: - template friend struct simfil::model_ptr; friend class TileFeatureLayer; friend class PointNode; friend class LinearRingNode; @@ -220,8 +219,14 @@ class Geometry final : public simfil::MandatoryDerivedModelNodeBase(key) {} + Geometry(Data* data, + ModelConstPtr pool, + ModelNodeAddress a, + simfil::detail::mp_key key); + Geometry() = delete; }; /** GeometryCollection node has `type` and `geometries` fields. */ @@ -229,7 +234,6 @@ class Geometry final : public simfil::MandatoryDerivedModelNodeBase { public: - template friend struct simfil::model_ptr; friend class TileFeatureLayer; friend class Feature; @@ -264,10 +268,13 @@ class GeometryCollection : public simfil::MandatoryDerivedModelNodeBase(key) {} + GeometryCollection(ModelConstPtr pool, ModelNodeAddress, simfil::detail::mp_key key); + GeometryCollection() = delete; +private: [[nodiscard]] ValueType type() const override; [[nodiscard]] ModelNode::Ptr at(int64_t) const override; [[nodiscard]] uint32_t size() const override; @@ -283,7 +290,6 @@ class GeometryCollection : public simfil::MandatoryDerivedModelNodeBase { public: - template friend struct simfil::model_ptr; friend class TileFeatureLayer; friend class Geometry; friend class MeshNode; @@ -299,9 +305,13 @@ class PointBufferNode final : public simfil::MandatoryDerivedModelNodeBase { public: - template friend struct simfil::model_ptr; friend class TileFeatureLayer; friend class Geometry; @@ -327,8 +336,8 @@ class PolygonNode final : public simfil::MandatoryDerivedModelNodeBase { public: - template friend struct simfil::model_ptr; friend class TileFeatureLayer; friend class Geometry; @@ -349,9 +357,13 @@ class MeshNode final : public simfil::MandatoryDerivedModelNodeBase { public: - template friend struct simfil::model_ptr; friend class TileFeatureLayer; friend class Geometry; @@ -370,9 +381,10 @@ class MeshTriangleCollectionNode : public simfil::MandatoryDerivedModelNodeBase< MeshTriangleCollectionNode() = delete; -private: - explicit MeshTriangleCollectionNode(const ModelNode& base); +public: + explicit MeshTriangleCollectionNode(const ModelNode& base, simfil::detail::mp_key key); +private: uint32_t index_ = 0; }; @@ -384,7 +396,6 @@ class MeshTriangleCollectionNode : public simfil::MandatoryDerivedModelNodeBase< class LinearRingNode : public simfil::MandatoryDerivedModelNodeBase { public: - template friend struct simfil::model_ptr; friend class TileFeatureLayer; friend class Geometry; @@ -397,9 +408,11 @@ class LinearRingNode : public simfil::MandatoryDerivedModelNodeBase length = {}); +public: + explicit LinearRingNode(const ModelNode& base, simfil::detail::mp_key key); + LinearRingNode(const ModelNode& base, std::optional length, simfil::detail::mp_key key); +private: model_ptr vertexBuffer() const; enum class Orientation : uint8_t { CW, CCW }; diff --git a/libs/model/include/mapget/model/pointnode.h b/libs/model/include/mapget/model/pointnode.h index 16b090b2..ebdf37c6 100644 --- a/libs/model/include/mapget/model/pointnode.h +++ b/libs/model/include/mapget/model/pointnode.h @@ -13,7 +13,6 @@ namespace mapget class PointNode final : public simfil::MandatoryDerivedModelNodeBase { public: - template friend struct simfil::model_ptr; friend class TileFeatureLayer; friend class Geometry; friend class PointBufferNode; @@ -27,22 +26,24 @@ class PointNode final : public simfil::MandatoryDerivedModelNodeBase bool Geometry::forEachPoint(LambdaType const& callback) const { - PointBufferNode vertexBufferNode{geomData_, model_, {ModelType::ColumnId::PointBuffers, addr_.index()}}; - for (auto i = 0; i < vertexBufferNode.size(); ++i) { - PointNode vertex{*vertexBufferNode.at(i), vertexBufferNode.baseGeomData_}; - if (!callback(vertex.point_)) + auto vertexBufferNode = model_ptr::make( + geomData_, model_, ModelNodeAddress{ModelType::ColumnId::PointBuffers, addr_.index()}); + for (auto i = 0; i < vertexBufferNode->size(); ++i) { + auto vertex = model_ptr::make(*vertexBufferNode->at(i), vertexBufferNode->baseGeomData_); + if (!callback(vertex->point_)) return false; } return true; } -} \ No newline at end of file +} diff --git a/libs/model/include/mapget/model/relation.h b/libs/model/include/mapget/model/relation.h index 6cd9df8e..89658c96 100644 --- a/libs/model/include/mapget/model/relation.h +++ b/libs/model/include/mapget/model/relation.h @@ -20,7 +20,6 @@ class Relation : public simfil::ProceduralObject<6, Relation, TileFeatureLayer> { friend class TileFeatureLayer; friend class Feature; - template friend struct simfil::model_ptr; public: /** @@ -72,9 +71,16 @@ class Relation : public simfil::ProceduralObject<6, Relation, TileFeatureLayer> } }; - Relation(Data* data, simfil::ModelConstPtr l, simfil::ModelNodeAddress a); - Relation() = default; +public: + explicit Relation(simfil::detail::mp_key key) + : simfil::ProceduralObject<6, Relation, TileFeatureLayer>(key) {} + Relation(Data* data, + simfil::ModelConstPtr l, + simfil::ModelNodeAddress a, + simfil::detail::mp_key key); + Relation() = delete; +protected: /** Reference to the actual data stored for the relation. */ Data* data_{}; }; diff --git a/libs/model/include/mapget/model/sourcedata.h b/libs/model/include/mapget/model/sourcedata.h index c7088c52..7c75f923 100644 --- a/libs/model/include/mapget/model/sourcedata.h +++ b/libs/model/include/mapget/model/sourcedata.h @@ -20,7 +20,6 @@ class SourceDataCompoundNode : public simfil::MandatoryDerivedModelNodeBase; public: SourceDataCompoundNode() = delete; @@ -52,9 +51,19 @@ class SourceDataCompoundNode : public simfil::MandatoryDerivedModelNodeBase(key), + data_(nullptr) {} + SourceDataCompoundNode(Data* data, + TileSourceDataLayer::ConstPtr model, + simfil::ModelNodeAddress address, + simfil::detail::mp_key key); + SourceDataCompoundNode(Data* data, + TileSourceDataLayer::Ptr model, + simfil::ModelNodeAddress address, + size_t initialSize, + simfil::detail::mp_key key); private: struct Data diff --git a/libs/model/include/mapget/model/sourcedatareference.h b/libs/model/include/mapget/model/sourcedatareference.h index 4f1ac6e2..61b3d33a 100644 --- a/libs/model/include/mapget/model/sourcedatareference.h +++ b/libs/model/include/mapget/model/sourcedatareference.h @@ -37,7 +37,6 @@ struct QualifiedSourceDataReference { class SourceDataReferenceCollection final : public simfil::MandatoryDerivedModelNodeBase { public: - template friend struct simfil::model_ptr; friend class TileFeatureLayer; ValueType type() const override; @@ -51,10 +50,17 @@ class SourceDataReferenceCollection final : public simfil::MandatoryDerivedModel */ void forEachReference(std::function fn) const; -private: - SourceDataReferenceCollection() = default; - SourceDataReferenceCollection(uint32_t offset, uint32_t size, ModelConstPtr pool, ModelNodeAddress a); +public: + explicit SourceDataReferenceCollection(simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(key) {} + SourceDataReferenceCollection(uint32_t offset, + uint32_t size, + ModelConstPtr pool, + ModelNodeAddress a, + simfil::detail::mp_key key); + SourceDataReferenceCollection() = delete; +private: uint32_t offset_ = {}; uint32_t size_ = {}; }; @@ -65,7 +71,6 @@ class SourceDataReferenceCollection final : public simfil::MandatoryDerivedModel class SourceDataReferenceItem final : public simfil::MandatoryDerivedModelNodeBase { public: - template friend struct simfil::model_ptr; friend class SourceDataReferenceCollection; friend class TileFeatureLayer; @@ -83,10 +88,16 @@ class SourceDataReferenceItem final : public simfil::MandatoryDerivedModelNodeBa std::string_view layerId() const; SourceDataAddress address() const; -private: - SourceDataReferenceItem() = default; - SourceDataReferenceItem(const QualifiedSourceDataReference* data, ModelConstPtr pool, ModelNodeAddress a); +public: + explicit SourceDataReferenceItem(simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(key) {} + SourceDataReferenceItem(const QualifiedSourceDataReference* data, + ModelConstPtr pool, + ModelNodeAddress a, + simfil::detail::mp_key key); + SourceDataReferenceItem() = delete; +private: const QualifiedSourceDataReference* const data_ = {}; }; diff --git a/libs/model/include/mapget/model/validity.h b/libs/model/include/mapget/model/validity.h index 218420f7..d941640b 100644 --- a/libs/model/include/mapget/model/validity.h +++ b/libs/model/include/mapget/model/validity.h @@ -14,8 +14,6 @@ class Geometry; class Validity : public simfil::ProceduralObject<6, Validity, TileFeatureLayer> { friend class TileFeatureLayer; - template - friend struct simfil::model_ptr; friend class PointNode; public: @@ -178,10 +176,16 @@ class Validity : public simfil::ProceduralObject<6, Validity, TileFeatureLayer> } }; -protected: - Validity(Data* data, simfil::ModelConstPtr layer, simfil::ModelNodeAddress a); - Validity() = default; +public: + explicit Validity(simfil::detail::mp_key key) + : simfil::ProceduralObject<6, Validity, TileFeatureLayer>(key) {} + Validity(Data* data, + simfil::ModelConstPtr layer, + simfil::ModelNodeAddress a, + simfil::detail::mp_key key); + Validity() = delete; +protected: /** * Pointer to the actual data stored for the attribute. */ @@ -194,8 +198,6 @@ class Validity : public simfil::ProceduralObject<6, Validity, TileFeatureLayer> struct MultiValidity : public simfil::BaseArray { friend class TileFeatureLayer; - template - friend struct simfil::model_ptr; /** * Append a new line position validity based on an absolute geographic position. @@ -280,4 +282,4 @@ struct MultiValidity : public simfil::BaseArray using simfil::BaseArray::BaseArray; }; -} \ No newline at end of file +} diff --git a/libs/model/src/attr.cpp b/libs/model/src/attr.cpp index 70c3c359..a48ea306 100644 --- a/libs/model/src/attr.cpp +++ b/libs/model/src/attr.cpp @@ -5,8 +5,12 @@ namespace mapget { -Attribute::Attribute(Attribute::Data* data, simfil::ModelConstPtr l, simfil::ModelNodeAddress a) - : simfil::ProceduralObject<2, Attribute, TileFeatureLayer>(data->fields_, std::move(l), a), data_(data) +Attribute::Attribute(Attribute::Data* data, + simfil::ModelConstPtr l, + simfil::ModelNodeAddress a, + simfil::detail::mp_key key) + : simfil::ProceduralObject<2, Attribute, TileFeatureLayer>(data->fields_, std::move(l), a, key), + data_(data) { if (data_->validities_) fields_.emplace_back( diff --git a/libs/model/src/attrlayer.cpp b/libs/model/src/attrlayer.cpp index 43f901df..86fe9e76 100644 --- a/libs/model/src/attrlayer.cpp +++ b/libs/model/src/attrlayer.cpp @@ -8,9 +8,10 @@ namespace mapget AttributeLayer::AttributeLayer( simfil::ArrayIndex i, simfil::ModelConstPtr l, - simfil::ModelNodeAddress a + simfil::ModelNodeAddress a, + simfil::detail::mp_key key ) - : simfil::Object(i, std::move(l), a) + : simfil::Object(i, std::move(l), a, key) { } @@ -46,9 +47,10 @@ bool AttributeLayer::forEachAttribute(const std::function(std::move(l), a), data_(&d) +Feature::Feature(Feature::Data& d, + simfil::ModelConstPtr l, + simfil::ModelNodeAddress a, + simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(std::move(l), a, key), + data_(&d) { updateFields(); } @@ -184,7 +188,9 @@ void Feature::updateFields() { fields_.clear(); // Add type field - fields_.emplace_back(StringPool::TypeStr, simfil::ValueNode(std::string_view("Feature"), model_)); + fields_.emplace_back( + StringPool::TypeStr, + simfil::model_ptr::make(std::string_view("Feature"), model_)); // Add id field fields_.emplace_back(StringPool::IdStr, Ptr::make(model_, data_->id_)); @@ -357,9 +363,11 @@ void Feature::setSourceDataReferences(simfil::ModelNode::Ptr const& addresses) Feature::FeaturePropertyView::FeaturePropertyView( Feature::Data& d, simfil::ModelConstPtr l, - simfil::ModelNodeAddress a + simfil::ModelNodeAddress a, + simfil::detail::mp_key key ) - : simfil::MandatoryDerivedModelNodeBase(std::move(l), a), data_(&d) + : simfil::MandatoryDerivedModelNodeBase(std::move(l), a, key), + data_(&d) { if (data_->attrs_) attrs_ = model().resolveObject(Ptr::make(model_, data_->attrs_)); diff --git a/libs/model/src/featureid.cpp b/libs/model/src/featureid.cpp index ebd3f877..79cf6dc9 100644 --- a/libs/model/src/featureid.cpp +++ b/libs/model/src/featureid.cpp @@ -6,8 +6,11 @@ namespace mapget { -FeatureId::FeatureId(FeatureId::Data& data, simfil::ModelConstPtr l, simfil::ModelNodeAddress a) - : simfil::MandatoryDerivedModelNodeBase(l, a), +FeatureId::FeatureId(FeatureId::Data& data, + simfil::ModelConstPtr l, + simfil::ModelNodeAddress a, + simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(l, a, key), data_(&data), fields_(model().resolveObject(Ptr::make(l, data_->idParts_))) { diff --git a/libs/model/src/featurelayer.cpp b/libs/model/src/featurelayer.cpp index e5cb6a56..9402d1b5 100644 --- a/libs/model/src/featurelayer.cpp +++ b/libs/model/src/featurelayer.cpp @@ -323,7 +323,8 @@ simfil::model_ptr TileFeatureLayer::newFeature( auto result = Feature( impl_->features_.back(), shared_from_this(), - simfil::ModelNodeAddress{ColumnId::Features, (uint32_t)featureIndex}); + simfil::ModelNodeAddress{ColumnId::Features, (uint32_t)featureIndex}, + mpKey_); // Add feature hash index entry. auto const& primaryIdComposition = getPrimaryIdComposition(typeId); @@ -368,7 +369,11 @@ TileFeatureLayer::newFeatureId( featureIdObject->addField(kk, x); }, v); } - return FeatureId(impl_->featureIds_.back(), shared_from_this(), {ColumnId::FeatureIds, (uint32_t)featureIdIndex}); + return FeatureId( + impl_->featureIds_.back(), + shared_from_this(), + {ColumnId::FeatureIds, (uint32_t)featureIdIndex}, + mpKey_); } model_ptr @@ -382,7 +387,11 @@ TileFeatureLayer::newRelation(const std::string_view& name, const model_ptraddr() }); - return Relation(&impl_->relations_.back(), shared_from_this(), {ColumnId::Relations, (uint32_t)relationIndex}); + return Relation( + &impl_->relations_.back(), + shared_from_this(), + {ColumnId::Relations, (uint32_t)relationIndex}, + mpKey_); } model_ptr TileFeatureLayer::getIdPrefix() @@ -407,7 +416,8 @@ TileFeatureLayer::newAttribute(const std::string_view& name, size_t initialCapac return Attribute( &impl_->attributes_.back(), shared_from_this(), - {ColumnId::Attributes, (uint32_t)attrIndex}); + {ColumnId::Attributes, (uint32_t)attrIndex}, + mpKey_); } model_ptr TileFeatureLayer::newAttributeLayer(size_t initialCapacity) @@ -417,7 +427,8 @@ model_ptr TileFeatureLayer::newAttributeLayer(size_t initialCapa return AttributeLayer( impl_->attrLayers_.back(), shared_from_this(), - {ColumnId::AttributeLayers, (uint32_t)layerIndex}); + {ColumnId::AttributeLayers, (uint32_t)layerIndex}, + mpKey_); } model_ptr TileFeatureLayer::newAttributeLayers(size_t initialCapacity) @@ -427,7 +438,8 @@ model_ptr TileFeatureLayer::newAttributeLayers(size_t initia return AttributeLayerList( impl_->attrLayerLists_.back(), shared_from_this(), - {ColumnId::AttributeLayerLists, (uint32_t)listIndex}); + {ColumnId::AttributeLayerLists, (uint32_t)listIndex}, + mpKey_); } model_ptr TileFeatureLayer::newGeometryCollection(size_t initialCapacity) @@ -435,7 +447,8 @@ model_ptr TileFeatureLayer::newGeometryCollection(size_t ini auto listIndex = arrayMemberStorage().new_array(initialCapacity); return GeometryCollection( shared_from_this(), - {ColumnId::GeometryCollections, (uint32_t)listIndex}); + {ColumnId::GeometryCollections, (uint32_t)listIndex}, + mpKey_); } model_ptr TileFeatureLayer::newGeometry(GeomType geomType, size_t initialCapacity) @@ -445,7 +458,8 @@ model_ptr TileFeatureLayer::newGeometry(GeomType geomType, size_t init return Geometry( &impl_->geom_.back(), shared_from_this(), - {ColumnId::Geometries, (uint32_t)impl_->geom_.size() - 1}); + {ColumnId::Geometries, (uint32_t)impl_->geom_.size() - 1}, + mpKey_); } model_ptr TileFeatureLayer::newGeometryView( @@ -458,7 +472,8 @@ model_ptr TileFeatureLayer::newGeometryView( return Geometry( &impl_->geom_.back(), shared_from_this(), - {ColumnId::Geometries, (uint32_t)impl_->geom_.size() - 1}); + {ColumnId::Geometries, (uint32_t)impl_->geom_.size() - 1}, + mpKey_); } model_ptr TileFeatureLayer::newSourceDataReferenceCollection(std::span list) @@ -471,7 +486,8 @@ model_ptr TileFeatureLayer::newSourceDataReferenc return { SourceDataReferenceCollection(index, size, shared_from_this(), - ModelNodeAddress(ColumnId::SourceDataReferenceCollections, sourceDataAddressListToModelAddress(index, size)))}; + ModelNodeAddress(ColumnId::SourceDataReferenceCollections, sourceDataAddressListToModelAddress(index, size)), + mpKey_)}; } model_ptr TileFeatureLayer::newValidity() @@ -480,7 +496,8 @@ model_ptr TileFeatureLayer::newValidity() return Validity( &impl_->validities_.back(), shared_from_this(), - {ColumnId::Validities, (uint32_t)impl_->validities_.size() - 1}); + {ColumnId::Validities, (uint32_t)impl_->validities_.size() - 1}, + mpKey_); } model_ptr TileFeatureLayer::newValidityCollection(size_t initialCapacity) @@ -488,7 +505,8 @@ model_ptr TileFeatureLayer::newValidityCollection(size_t initialC auto validityArrId = arrayMemberStorage().new_array(initialCapacity); return MultiValidity( shared_from_this(), - {ColumnId::ValidityCollections, (uint32_t)validityArrId}); + {ColumnId::ValidityCollections, (uint32_t)validityArrId}, + mpKey_); } model_ptr TileFeatureLayer::resolveAttributeLayer(simfil::ModelNode const& n) const @@ -498,7 +516,8 @@ model_ptr TileFeatureLayer::resolveAttributeLayer(simfil::ModelN return AttributeLayer( impl_->attrLayers_[n.addr().index()], shared_from_this(), - n.addr()); + n.addr(), + mpKey_); } model_ptr TileFeatureLayer::resolveAttributeLayerList(simfil::ModelNode const& n) const @@ -508,7 +527,8 @@ model_ptr TileFeatureLayer::resolveAttributeLayerList(simfil return AttributeLayerList( impl_->attrLayerLists_[n.addr().index()], shared_from_this(), - n.addr()); + n.addr(), + mpKey_); } model_ptr TileFeatureLayer::resolveAttribute(simfil::ModelNode const& n) const @@ -518,7 +538,8 @@ model_ptr TileFeatureLayer::resolveAttribute(simfil::ModelNode const& return Attribute( &impl_->attributes_[n.addr().index()], shared_from_this(), - n.addr()); + n.addr(), + mpKey_); } model_ptr TileFeatureLayer::resolveFeature(simfil::ModelNode const& n) const @@ -528,7 +549,8 @@ model_ptr TileFeatureLayer::resolveFeature(simfil::ModelNode const& n) return Feature( impl_->features_[n.addr().index()], shared_from_this(), - n.addr()); + n.addr(), + mpKey_); } model_ptr TileFeatureLayer::resolveFeatureId(simfil::ModelNode const& n) const @@ -538,7 +560,8 @@ model_ptr TileFeatureLayer::resolveFeatureId(simfil::ModelNode const& return FeatureId( impl_->featureIds_[n.addr().index()], shared_from_this(), - n.addr()); + n.addr(), + mpKey_); } model_ptr TileFeatureLayer::resolveRelation(const simfil::ModelNode& n) const @@ -548,7 +571,8 @@ model_ptr TileFeatureLayer::resolveRelation(const simfil::ModelNode& n return Relation( &impl_->relations_[n.addr().index()], shared_from_this(), - n.addr()); + n.addr(), + mpKey_); } model_ptr TileFeatureLayer::resolvePoint(const simfil::ModelNode& n) const @@ -556,7 +580,7 @@ model_ptr TileFeatureLayer::resolvePoint(const simfil::ModelNode& n) if (n.addr().column() != ColumnId::Points) raise("Cannot cast this node to a Point."); return PointNode( - n, &impl_->geom_.at(n.addr().index())); + n, &impl_->geom_.at(n.addr().index()), mpKey_); } model_ptr TileFeatureLayer::resolveValidityPoint(const simfil::ModelNode& n) const @@ -564,7 +588,7 @@ model_ptr TileFeatureLayer::resolveValidityPoint(const simfil::ModelN if (n.addr().column() != ColumnId::ValidityPoints) raise("Cannot cast this node to a ValidityPoint."); return PointNode( - n, &impl_->validities_.at(n.addr().index())); + n, &impl_->validities_.at(n.addr().index()), mpKey_); } model_ptr TileFeatureLayer::resolveValidity(simfil::ModelNode const& n) const @@ -574,7 +598,8 @@ model_ptr TileFeatureLayer::resolveValidity(simfil::ModelNode const& n return Validity( &impl_->validities_[n.addr().index()], shared_from_this(), - n.addr()); + n.addr(), + mpKey_); } model_ptr TileFeatureLayer::resolveValidityCollection(const simfil::ModelNode& n) const @@ -583,7 +608,8 @@ model_ptr TileFeatureLayer::resolveValidityCollection(const simfi raise("Cannot cast this node to a ValidityCollection."); return MultiValidity( shared_from_this(), - n.addr()); + n.addr(), + mpKey_); } model_ptr TileFeatureLayer::resolvePointBuffer(const simfil::ModelNode& n) const @@ -591,14 +617,16 @@ model_ptr TileFeatureLayer::resolvePointBuffer(const simfil::Mo return PointBufferNode( &impl_->geom_.at(n.addr().index()), shared_from_this(), - n.addr()); + n.addr(), + mpKey_); } model_ptr TileFeatureLayer::resolvePolygon(const simfil::ModelNode& n) const { return PolygonNode( shared_from_this(), - n.addr()); + n.addr(), + mpKey_); } model_ptr TileFeatureLayer::resolveMesh(const simfil::ModelNode& n) const @@ -606,12 +634,13 @@ model_ptr TileFeatureLayer::resolveMesh(const simfil::ModelNode& n) co return MeshNode( &impl_->geom_.at(n.addr().index()), shared_from_this(), - n.addr()); + n.addr(), + mpKey_); } model_ptr TileFeatureLayer::resolveLinearRing(const simfil::ModelNode& n) const { - return LinearRingNode(n); + return LinearRingNode(n, mpKey_); } model_ptr TileFeatureLayer::resolveGeometry(const simfil::ModelNode& n) const @@ -619,24 +648,25 @@ model_ptr TileFeatureLayer::resolveGeometry(const simfil::ModelNode& n return Geometry( &const_cast(impl_->geom_.at(n.addr().index())), // FIXME: const_cast?! shared_from_this(), - n.addr()); + n.addr(), + mpKey_); } model_ptr TileFeatureLayer::resolveMeshTriangleLinearRing(const simfil::ModelNode& n) const { - return LinearRingNode(n, 3); + return LinearRingNode(n, 3, mpKey_); } model_ptr TileFeatureLayer::resolveMeshTriangleCollection(const simfil::ModelNode& n) const { - return MeshTriangleCollectionNode(n); + return MeshTriangleCollectionNode(n, mpKey_); } model_ptr TileFeatureLayer::resolveGeometryCollection(const simfil::ModelNode& n) const { return GeometryCollection( - shared_from_this(), n.addr()); + shared_from_this(), n.addr(), mpKey_); } model_ptr @@ -647,7 +677,7 @@ TileFeatureLayer::resolveSourceDataReferenceCollection(const simfil::ModelNode& auto [index, size] = modelAddressToSourceDataAddressList(n.addr().index()); const auto& data = impl_->sourceDataReferences_; - return SourceDataReferenceCollection(index, size, shared_from_this(), n.addr()); + return SourceDataReferenceCollection(index, size, shared_from_this(), n.addr(), mpKey_); } model_ptr @@ -657,7 +687,7 @@ TileFeatureLayer::resolveSourceDataReferenceItem(const simfil::ModelNode& n) con raise("Cannot cast this node to an SourceDataReferenceItem."); const auto* data = &impl_->sourceDataReferences_.at(n.addr().index()); - return SourceDataReferenceItem(data, shared_from_this(), n.addr()); + return SourceDataReferenceItem(data, shared_from_this(), n.addr(), mpKey_); } tl::expected TileFeatureLayer::resolve(const simfil::ModelNode& n, const simfil::Model::ResolveFn& cb) const @@ -671,7 +701,8 @@ tl::expected TileFeatureLayer::resolve(const simfil::ModelN cb(Feature::FeaturePropertyView( impl_->features_[n.addr().index()], shared_from_this(), - n.addr() + n.addr(), + mpKey_ )); return {}; case ColumnId::FeatureIds: diff --git a/libs/model/src/geometry.cpp b/libs/model/src/geometry.cpp index d0a59d30..bf6d5444 100644 --- a/libs/model/src/geometry.cpp +++ b/libs/model/src/geometry.cpp @@ -62,8 +62,8 @@ using namespace simfil; /** Model node impls. for GeometryCollection */ -GeometryCollection::GeometryCollection(ModelConstPtr pool_, ModelNodeAddress a) - : simfil::MandatoryDerivedModelNodeBase(std::move(pool_), a) +GeometryCollection::GeometryCollection(ModelConstPtr pool_, ModelNodeAddress a, simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(std::move(pool_), a, key) {} ValueType GeometryCollection::type() const { @@ -73,7 +73,7 @@ ValueType GeometryCollection::type() const { ModelNode::Ptr GeometryCollection::at(int64_t i) const { if (auto singleGeomEntry = singleGeom()) return singleGeomEntry->at(i); - if (i == 0) return ValueNode(GeometryCollectionStr, model_); + if (i == 0) return model_ptr::make(GeometryCollectionStr, model_); if (i == 1) return ModelNode::Ptr::make(model_, ModelNodeAddress{simfil::ModelPool::Arrays, addr_.index()}); throw std::out_of_range("geom collection: Out of range."); } @@ -139,8 +139,9 @@ size_t GeometryCollection::numGeometries() const /** ModelNode impls. for Geometry */ -Geometry::Geometry(Data* data, ModelConstPtr pool_, ModelNodeAddress a) - : simfil::MandatoryDerivedModelNodeBase(std::move(pool_), a), geomData_(data) +Geometry::Geometry(Data* data, ModelConstPtr pool_, ModelNodeAddress a, simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(std::move(pool_), a, key), + geomData_(data) { storage_ = &model().vertexBufferStorage(); } @@ -196,7 +197,7 @@ ModelNode::Ptr Geometry::get(const StringId& f) const { return ModelNode::Ptr::make(model_, geomData_->sourceDataReferences_); } if (f == StringPool::TypeStr) { - return ValueNode( + return model_ptr::make( geomData_->type_ == GeomType::Points ? MultiPointStr : geomData_->type_ == GeomType::Line ? LineStringStr : geomData_->type_ == GeomType::Polygon ? PolygonStr : @@ -290,15 +291,17 @@ bool Geometry::iterate(const IterCallback& cb) const size_t Geometry::numPoints() const { - PointBufferNode vertexBufferNode{geomData_, model_, {TileFeatureLayer::ColumnId::PointBuffers, addr_.index()}}; - return vertexBufferNode.size(); + auto vertexBufferNode = model_ptr::make( + geomData_, model_, ModelNodeAddress{TileFeatureLayer::ColumnId::PointBuffers, addr_.index()}); + return vertexBufferNode->size(); } Point Geometry::pointAt(size_t index) const { - PointBufferNode vertexBufferNode{geomData_, model_, {TileFeatureLayer::ColumnId::PointBuffers, addr_.index()}}; - PointNode vertex{*vertexBufferNode.at((int64_t)index), vertexBufferNode.baseGeomData_}; - return vertex.point_; + auto vertexBufferNode = model_ptr::make( + geomData_, model_, ModelNodeAddress{TileFeatureLayer::ColumnId::PointBuffers, addr_.index()}); + auto vertex = model_ptr::make(*vertexBufferNode->at((int64_t)index), vertexBufferNode->baseGeomData_); + return vertex->point_; } std::optional Geometry::name() const @@ -473,8 +476,8 @@ Point Geometry::percentagePositionFromGeometries(std::vector /** ModelNode impls. for PolygonNode */ -PolygonNode::PolygonNode(ModelConstPtr pool, ModelNodeAddress const& a) - : simfil::MandatoryDerivedModelNodeBase(std::move(pool), a) +PolygonNode::PolygonNode(ModelConstPtr pool, ModelNodeAddress const& a, simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(std::move(pool), a, key) {} ValueType PolygonNode::type() const @@ -515,13 +518,17 @@ bool PolygonNode::iterate(IterCallback const& cb) const /** ModelNode impls. for MeshNode */ -MeshNode::MeshNode(Geometry::Data const* geomData, ModelConstPtr pool, ModelNodeAddress const& a) - : simfil::MandatoryDerivedModelNodeBase(std::move(pool), a), geomData_(geomData) +MeshNode::MeshNode(Geometry::Data const* geomData, + ModelConstPtr pool, + ModelNodeAddress const& a, + simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(std::move(pool), a, key), + geomData_(geomData) { - auto vertex_buffer = PointBufferNode{ - geomData_, model_, {TileFeatureLayer::ColumnId::PointBuffers, addr_.index()}}; - assert(vertex_buffer.size() % 3 == 0); - size_ = vertex_buffer.size() / 3; + auto vertex_buffer = model_ptr::make( + geomData_, model_, ModelNodeAddress{TileFeatureLayer::ColumnId::PointBuffers, addr_.index()}); + assert(vertex_buffer->size() % 3 == 0); + size_ = vertex_buffer->size() / 3; } ValueType MeshNode::type() const @@ -551,9 +558,9 @@ bool MeshNode::iterate(IterCallback const& cb) const return true; } -MeshTriangleCollectionNode::MeshTriangleCollectionNode(const ModelNode& base) - : simfil::MandatoryDerivedModelNodeBase(base) - , index_(std::get(data_) * 3) +MeshTriangleCollectionNode::MeshTriangleCollectionNode(const ModelNode& base, simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(base, key), + index_(std::get(data_) * 3) {} ValueType MeshTriangleCollectionNode::type() const @@ -583,8 +590,12 @@ bool MeshTriangleCollectionNode::iterate(IterCallback const& cb) const /** ModelNode impls. for LinearRingNode (a closed, simple polygon in CCW order) */ -LinearRingNode::LinearRingNode(const ModelNode& base, std::optional length) - : simfil::MandatoryDerivedModelNodeBase(base) +LinearRingNode::LinearRingNode(const ModelNode& base, simfil::detail::mp_key key) + : LinearRingNode(base, std::optional{}, key) +{} + +LinearRingNode::LinearRingNode(const ModelNode& base, std::optional length, simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(base, key) { if (std::get_if(&data_)) offset_ = std::get(data_); @@ -694,8 +705,13 @@ model_ptr LinearRingNode::vertexBuffer() const /** ModelNode impls. for VertexBufferNode */ -PointBufferNode::PointBufferNode(Geometry::Data const* geomData, ModelConstPtr pool_, ModelNodeAddress const& a) - : simfil::MandatoryDerivedModelNodeBase(std::move(pool_), a), baseGeomData_(geomData), baseGeomAddress_(a) +PointBufferNode::PointBufferNode(Geometry::Data const* geomData, + ModelConstPtr pool_, + ModelNodeAddress const& a, + simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(std::move(pool_), a, key), + baseGeomData_(geomData), + baseGeomAddress_(a) { storage_ = &model().vertexBufferStorage(); @@ -762,8 +778,8 @@ bool PointBufferNode::iterate(const IterCallback& cb) const Point PointBufferNode::pointAt(int64_t index) const { - PointNode vertex{*at(index), baseGeomData_}; - return vertex.point_; + auto vertex = model_ptr::make(*at(index), baseGeomData_); + return vertex->point_; } } diff --git a/libs/model/src/pointnode.cpp b/libs/model/src/pointnode.cpp index abb0a531..ff6d7caf 100644 --- a/libs/model/src/pointnode.cpp +++ b/libs/model/src/pointnode.cpp @@ -9,8 +9,10 @@ namespace mapget /** Model node impls for VertexNode. */ -PointNode::PointNode(ModelNode const& baseNode, Geometry::Data const* geomData) - : simfil::MandatoryDerivedModelNodeBase(baseNode) +PointNode::PointNode(ModelNode const& baseNode, + Geometry::Data const* geomData, + simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(baseNode, key) { if (geomData->isView_) throw std::runtime_error("Point must be constructed through VertexBuffer which resolves view to geometry."); @@ -25,8 +27,10 @@ PointNode::PointNode(ModelNode const& baseNode, Geometry::Data const* geomData) } } -PointNode::PointNode(ModelNode const& baseNode, Validity::Data const* geomData) - : simfil::MandatoryDerivedModelNodeBase(baseNode) +PointNode::PointNode(ModelNode const& baseNode, + Validity::Data const* geomData, + simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(baseNode, key) { auto i = std::get(data_); // The extracted point index may point to a validity's single point @@ -71,10 +75,10 @@ StringId PointNode::keyAt(int64_t i) const { bool PointNode::iterate(const IterCallback& cb) const { - if (!cb(ValueNode(point_.x, model_))) return false; - if (!cb(ValueNode(point_.y, model_))) return false; - if (!cb(ValueNode(point_.z, model_))) return false; + if (!cb(*model_ptr::make(point_.x, model_))) return false; + if (!cb(*model_ptr::make(point_.y, model_))) return false; + if (!cb(*model_ptr::make(point_.z, model_))) return false; return true; } -} \ No newline at end of file +} diff --git a/libs/model/src/relation.cpp b/libs/model/src/relation.cpp index e686e3d6..a0319328 100644 --- a/libs/model/src/relation.cpp +++ b/libs/model/src/relation.cpp @@ -6,8 +6,12 @@ namespace mapget { -Relation::Relation(Relation::Data* data, simfil::ModelConstPtr l, simfil::ModelNodeAddress a) - : simfil::ProceduralObject<6, Relation, TileFeatureLayer>(std::move(l), a), data_(data) +Relation::Relation(Relation::Data* data, + simfil::ModelConstPtr l, + simfil::ModelNodeAddress a, + simfil::detail::mp_key key) + : simfil::ProceduralObject<6, Relation, TileFeatureLayer>(std::move(l), a, key), + data_(data) { fields_.emplace_back( StringPool::NameStr, diff --git a/libs/model/src/sourcedata.cpp b/libs/model/src/sourcedata.cpp index b9d9ebf8..a25af31b 100644 --- a/libs/model/src/sourcedata.cpp +++ b/libs/model/src/sourcedata.cpp @@ -94,14 +94,23 @@ bool SourceDataCompoundNode::iterate(IterCallback const& cb) const return true; } -SourceDataCompoundNode::SourceDataCompoundNode(Data* data, TileSourceDataLayer::ConstPtr model, simfil::ModelNodeAddress addr) - : simfil::MandatoryDerivedModelNodeBase(std::move(model), addr), data_(data) +SourceDataCompoundNode::SourceDataCompoundNode(Data* data, + TileSourceDataLayer::ConstPtr model, + simfil::ModelNodeAddress addr, + simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(std::move(model), addr, key), + data_(data) { assert(data_); } -SourceDataCompoundNode::SourceDataCompoundNode(Data* data, TileSourceDataLayer::Ptr model, simfil::ModelNodeAddress addr, size_t initialSize) - : simfil::MandatoryDerivedModelNodeBase(model, addr), data_(data) +SourceDataCompoundNode::SourceDataCompoundNode(Data* data, + TileSourceDataLayer::Ptr model, + simfil::ModelNodeAddress addr, + size_t initialSize, + simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(model, addr, key), + data_(data) { assert(data_); assert(!data_->object_); diff --git a/libs/model/src/sourcedatalayer.cpp b/libs/model/src/sourcedatalayer.cpp index b6d3e250..397a39f4 100644 --- a/libs/model/src/sourcedatalayer.cpp +++ b/libs/model/src/sourcedatalayer.cpp @@ -96,7 +96,8 @@ model_ptr TileSourceDataLayer::newCompound(size_t initia &data, std::static_pointer_cast(shared_from_this()), ModelNodeAddress(Compound, static_cast(index)), - initialSize); + initialSize, + mpKey_); } model_ptr TileSourceDataLayer::resolveCompound(simfil::ModelNode const& n) const @@ -104,7 +105,11 @@ model_ptr TileSourceDataLayer::resolveCompound(simfil::M assert(n.addr().column() == Compound && "Unexpected column type!"); auto& data = impl_->compounds_.at(n.addr().index()); - return SourceDataCompoundNode(&data, std::static_pointer_cast(shared_from_this()), n.addr()); + return SourceDataCompoundNode( + &data, + std::static_pointer_cast(shared_from_this()), + n.addr(), + mpKey_); } tl::expected TileSourceDataLayer::resolve(const simfil::ModelNode& n, const ResolveFn& cb) const diff --git a/libs/model/src/sourcedatareference.cpp b/libs/model/src/sourcedatareference.cpp index 95b771ea..17e002ea 100644 --- a/libs/model/src/sourcedatareference.cpp +++ b/libs/model/src/sourcedatareference.cpp @@ -45,8 +45,14 @@ void SourceDataReferenceCollection::forEachReference(std::function(pool, a), offset_(offset), size_(size) +SourceDataReferenceCollection::SourceDataReferenceCollection(uint32_t offset, + uint32_t size, + ModelConstPtr pool, + ModelNodeAddress a, + simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(pool, a, key), + offset_(offset), + size_(size) {} ValueType SourceDataReferenceItem::type() const @@ -120,8 +126,12 @@ SourceDataAddress SourceDataReferenceItem::address() const return data_->reference_.address_; } -SourceDataReferenceItem::SourceDataReferenceItem(const QualifiedSourceDataReference* const data, const ModelConstPtr pool, const ModelNodeAddress a) - : simfil::MandatoryDerivedModelNodeBase(pool, a), data_(data) +SourceDataReferenceItem::SourceDataReferenceItem(const QualifiedSourceDataReference* const data, + const ModelConstPtr pool, + const ModelNodeAddress a, + simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(pool, a, key), + data_(data) {} } diff --git a/libs/model/src/validity.cpp b/libs/model/src/validity.cpp index 5ad07f0a..0fd63a23 100644 --- a/libs/model/src/validity.cpp +++ b/libs/model/src/validity.cpp @@ -41,8 +41,10 @@ void Validity::setFeatureId(model_ptr featureId) Validity::Validity(Validity::Data* data, simfil::ModelConstPtr layer, - simfil::ModelNodeAddress a) - : simfil::ProceduralObject<6, Validity, TileFeatureLayer>(std::move(layer), a), data_(data) + simfil::ModelNodeAddress a, + simfil::detail::mp_key key) + : simfil::ProceduralObject<6, Validity, TileFeatureLayer>(std::move(layer), a, key), + data_(data) { if (data_->direction_) fields_.emplace_back( From eaff0d775db2374413d8cd6af5dee2b94e8ab163 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Fri, 6 Feb 2026 12:00:21 +0100 Subject: [PATCH 28/38] Support less verbose generic Model::resolve. --- cmake/deps.cmake | 2 +- libs/model/include/mapget/model/attrlayer.h | 14 +- .../model/include/mapget/model/featurelayer.h | 40 +- libs/model/include/mapget/model/geometry.h | 17 +- libs/model/include/mapget/model/pointnode.h | 3 + .../include/mapget/model/sourcedatalayer.h | 17 +- libs/model/src/attr.cpp | 5 +- libs/model/src/attrlayer.cpp | 12 +- libs/model/src/feature.cpp | 28 +- libs/model/src/featureid.cpp | 2 +- libs/model/src/featurelayer.cpp | 373 ++++++++++-------- libs/model/src/geometry.cpp | 61 +-- libs/model/src/relation.cpp | 20 +- libs/model/src/sourcedata.cpp | 2 +- libs/model/src/sourcedatalayer.cpp | 19 +- libs/model/src/sourcedatareference.cpp | 6 +- libs/model/src/validity.cpp | 10 +- 17 files changed, 350 insertions(+), 281 deletions(-) diff --git a/cmake/deps.cmake b/cmake/deps.cmake index 323dc283..d9d1fe05 100644 --- a/cmake/deps.cmake +++ b/cmake/deps.cmake @@ -15,7 +15,7 @@ CPMAddPackage( "EXPECTED_BUILD_TESTS OFF" "EXPECTED_BUILD_PACKAGE_DEB OFF") CPMAddPackage( - URI "gh:Klebert-Engineering/simfil@0.6.3#v0.6.3" + URI "gh:Klebert-Engineering/simfil@0.6.3#improvement/generic-resolve" OPTIONS "SIMFIL_WITH_MODEL_JSON ON" "SIMFIL_SHARED OFF") diff --git a/libs/model/include/mapget/model/attrlayer.h b/libs/model/include/mapget/model/attrlayer.h index e51db816..7c205594 100644 --- a/libs/model/include/mapget/model/attrlayer.h +++ b/libs/model/include/mapget/model/attrlayer.h @@ -14,12 +14,15 @@ class AttributeLayerList; * as speed limits, might belong to the same attribute layer. * TODO: Convert to use BaseObject */ -class AttributeLayer : public simfil::Object +class AttributeLayer : public simfil::BaseObject { friend class TileFeatureLayer; friend class bitsery::Access; public: + using BaseObject::addField; + using BaseObject::get; + /** * Create a new attribute and immediately insert it into the layer. */ @@ -40,7 +43,7 @@ class AttributeLayer : public simfil::Object bool forEachAttribute(std::function const& attr)> const& cb) const; public: - explicit AttributeLayer(simfil::detail::mp_key key) : simfil::Object(key) {} + explicit AttributeLayer(simfil::detail::mp_key key) : BaseObject(key) {} AttributeLayer(simfil::ArrayIndex i, simfil::ModelConstPtr l, simfil::ModelNodeAddress a, @@ -53,13 +56,16 @@ class AttributeLayer : public simfil::Object * stores (layer-name, layer) pairs. * TODO: Convert to use BaseObject */ -class AttributeLayerList : public simfil::Object +class AttributeLayerList : public simfil::BaseObject { friend class TileFeatureLayer; friend class bitsery::Access; friend class Feature; public: + using BaseObject::addField; + using BaseObject::get; + /** * Create a new named layer and immediately insert it into the collection. */ @@ -81,7 +87,7 @@ class AttributeLayerList : public simfil::Object ) const; public: - explicit AttributeLayerList(simfil::detail::mp_key key) : simfil::Object(key) {} + explicit AttributeLayerList(simfil::detail::mp_key key) : BaseObject(key) {} AttributeLayerList(simfil::ArrayIndex i, simfil::ModelConstPtr l, simfil::ModelNodeAddress a, diff --git a/libs/model/include/mapget/model/featurelayer.h b/libs/model/include/mapget/model/featurelayer.h index 07888c60..fb7dc307 100644 --- a/libs/model/include/mapget/model/featurelayer.h +++ b/libs/model/include/mapget/model/featurelayer.h @@ -49,8 +49,16 @@ class TileFeatureLayer : public TileLayer, public simfil::ModelPool friend class SourceDataReferenceCollection; friend class SourceDataReferenceItem; friend class Validity; + template + friend model_ptr resolveInternal( + simfil::res::tag, + TileFeatureLayer const&, + simfil::ModelNode const&); public: + // Keep ModelPool::resolve overloads visible alongside the override below. + using ModelPool::resolve; + /** * This constructor initializes a new TileFeatureLayer instance. * Each instance is associated with a specific TileId, nodeId, and mapId. @@ -297,30 +305,6 @@ class TileFeatureLayer : public TileLayer, public simfil::ModelPool TileFeatureLayer::Ptr const& otherLayer, simfil::ModelNode::Ptr const& otherNode); - /** - * Node resolution functions. - */ - model_ptr resolveAttributeLayer(simfil::ModelNode const& n) const; - model_ptr resolveAttributeLayerList(simfil::ModelNode const& n) const; - model_ptr resolveAttribute(simfil::ModelNode const& n) const; - model_ptr resolveFeature(simfil::ModelNode const& n) const; - model_ptr resolveFeatureId(simfil::ModelNode const& n) const; - model_ptr resolveRelation(simfil::ModelNode const& n) const; - model_ptr resolvePoint(const simfil::ModelNode& n) const; - model_ptr resolvePointBuffer(const simfil::ModelNode& n) const; - model_ptr resolveGeometry(simfil::ModelNode const& n) const; - model_ptr resolveGeometryCollection(simfil::ModelNode const& n) const; - model_ptr resolveMesh(simfil::ModelNode const& n) const; - model_ptr resolveMeshTriangleCollection(simfil::ModelNode const& n) const; - model_ptr resolveMeshTriangleLinearRing(simfil::ModelNode const& n) const; - model_ptr resolvePolygon(simfil::ModelNode const& n) const; - model_ptr resolveLinearRing(simfil::ModelNode const& n) const; - model_ptr resolveSourceDataReferenceCollection(simfil::ModelNode const& n) const; - model_ptr resolveSourceDataReferenceItem(simfil::ModelNode const& n) const; - model_ptr resolveValidityPoint(const simfil::ModelNode& n) const; - model_ptr resolveValidity(simfil::ModelNode const& n) const; - model_ptr resolveValidityCollection(simfil::ModelNode const& n) const; - /** * The ColumnId enum provides identifiers for different * types of columns that can be associated with feature data. @@ -369,4 +353,12 @@ class TileFeatureLayer : public TileLayer, public simfil::ModelPool struct Impl; std::unique_ptr impl_; }; + +// Primary template for ADL-based resolve hooks (specialized in featurelayer.cpp). +template +simfil::model_ptr resolveInternal( + simfil::res::tag, + TileFeatureLayer const& model, + simfil::ModelNode const& node); + } diff --git a/libs/model/include/mapget/model/geometry.h b/libs/model/include/mapget/model/geometry.h index f0de2f29..45aa72c4 100644 --- a/libs/model/include/mapget/model/geometry.h +++ b/libs/model/include/mapget/model/geometry.h @@ -264,7 +264,7 @@ class GeometryCollection : public simfil::MandatoryDerivedModelNodeBase()->arrayMemberStorage().range((simfil::ArrayIndex)addr().index()); return std::all_of(geomArray.begin(), geomArray.end(), [this, &callback](auto&& geomNodeAddress){ - return callback(modelPtr()->resolveGeometry(*ModelNode::Ptr::make(model_, geomNodeAddress))); + return callback(modelPtr()->template resolve(geomNodeAddress)); }); } @@ -294,6 +294,9 @@ class PointBufferNode final : public simfil::MandatoryDerivedModelNodeBase(key) {} + [[nodiscard]] ValueType type() const override; [[nodiscard]] ModelNode::Ptr at(int64_t) const override; [[nodiscard]] uint32_t size() const override; @@ -327,6 +330,9 @@ class PolygonNode final : public simfil::MandatoryDerivedModelNodeBase(key) {} + [[nodiscard]] ValueType type() const override; [[nodiscard]] ModelNode::Ptr at(int64_t) const override; [[nodiscard]] uint32_t size() const override; @@ -348,6 +354,9 @@ class MeshNode final : public simfil::MandatoryDerivedModelNodeBase(key) {} + [[nodiscard]] ValueType type() const override; [[nodiscard]] ModelNode::Ptr at(int64_t) const override; [[nodiscard]] uint32_t size() const override; @@ -374,6 +383,9 @@ class MeshTriangleCollectionNode : public simfil::MandatoryDerivedModelNodeBase< friend class TileFeatureLayer; friend class Geometry; + explicit MeshTriangleCollectionNode(simfil::detail::mp_key key) + : simfil::MandatoryDerivedModelNodeBase(key) {} + [[nodiscard]] ValueType type() const override; [[nodiscard]] ModelNode::Ptr at(int64_t) const override; [[nodiscard]] uint32_t size() const override; @@ -399,6 +411,9 @@ class LinearRingNode : public simfil::MandatoryDerivedModelNodeBase(key) {} + [[nodiscard]] ValueType type() const override; [[nodiscard]] ModelNode::Ptr at(int64_t) const override; [[nodiscard]] uint32_t size() const override; diff --git a/libs/model/include/mapget/model/pointnode.h b/libs/model/include/mapget/model/pointnode.h index ebdf37c6..856185e3 100644 --- a/libs/model/include/mapget/model/pointnode.h +++ b/libs/model/include/mapget/model/pointnode.h @@ -17,6 +17,9 @@ class PointNode final : public simfil::MandatoryDerivedModelNodeBase(key) {} + [[nodiscard]] ValueType type() const override; [[nodiscard]] ModelNode::Ptr at(int64_t) const override; [[nodiscard]] uint32_t size() const override; diff --git a/libs/model/include/mapget/model/sourcedatalayer.h b/libs/model/include/mapget/model/sourcedatalayer.h index 4cb58928..8fc44a97 100644 --- a/libs/model/include/mapget/model/sourcedatalayer.h +++ b/libs/model/include/mapget/model/sourcedatalayer.h @@ -16,12 +16,21 @@ class SourceDataCompoundNode; class TileSourceDataLayer : public TileLayer, public simfil::ModelPool { public: + // Keep ModelPool::resolve overloads visible alongside the override below. + using ModelPool::resolve; + using Ptr = std::shared_ptr; using ConstPtr = std::shared_ptr; template using model_ptr = simfil::model_ptr; + template + friend model_ptr resolveInternal( + simfil::res::tag, + TileSourceDataLayer const&, + simfil::ModelNode const&); + /** * ModelPool colunm ids */ @@ -47,7 +56,6 @@ class TileSourceDataLayer : public TileLayer, public simfil::ModelPool * Node factory interface */ model_ptr newCompound(size_t initialSize); - model_ptr resolveCompound(simfil::ModelNode const&) const; /** * Get this pool's simfil evaluation environment. @@ -88,4 +96,11 @@ class TileSourceDataLayer : public TileLayer, public simfil::ModelPool std::unique_ptr impl_; }; +// Primary template for ADL-based resolve hooks (specialized in sourcedatalayer.cpp). +template +simfil::model_ptr resolveInternal( + simfil::res::tag, + TileSourceDataLayer const& model, + simfil::ModelNode const& node); + } diff --git a/libs/model/src/attr.cpp b/libs/model/src/attr.cpp index a48ea306..f8710426 100644 --- a/libs/model/src/attr.cpp +++ b/libs/model/src/attr.cpp @@ -52,7 +52,8 @@ bool Attribute::forEachField( model_ptr Attribute::sourceDataReferences() const { if (data_->sourceDataRefs_) { - return model().resolveSourceDataReferenceCollection(*model_ptr::make(model_, data_->sourceDataRefs_)); + return model().resolve( + *model_ptr::make(model_, data_->sourceDataRefs_)); } return {}; } @@ -77,7 +78,7 @@ model_ptr Attribute::validityOrNull() const if (!data_->validities_) { return {}; } - return model().resolveValidityCollection(*ModelNode::Ptr::make(model_, data_->validities_)); + return model().resolve(data_->validities_); } void Attribute::setValidity(const model_ptr& validities) const diff --git a/libs/model/src/attrlayer.cpp b/libs/model/src/attrlayer.cpp index 86fe9e76..3cd83cc4 100644 --- a/libs/model/src/attrlayer.cpp +++ b/libs/model/src/attrlayer.cpp @@ -11,7 +11,7 @@ AttributeLayer::AttributeLayer( simfil::ModelNodeAddress a, simfil::detail::mp_key key ) - : simfil::Object(i, std::move(l), a, key) + : simfil::BaseObject(i, std::move(l), a, key) { } @@ -25,7 +25,7 @@ AttributeLayer::newAttribute(const std::string_view& name, size_t initialCapacit void AttributeLayer::addAttribute(model_ptr a) { - addField(a->name(), simfil::ModelNode::Ptr(a)); + addField(a->name(), a); } bool AttributeLayer::forEachAttribute(const std::function&)>& cb) const @@ -37,7 +37,7 @@ bool AttributeLayer::forEachAttribute(const std::function(model()).resolveAttribute(*value); + auto attr = static_cast(model()).resolve(*value); if (!cb(attr)) return false; } @@ -50,7 +50,7 @@ AttributeLayerList::AttributeLayerList( simfil::ModelNodeAddress a, simfil::detail::mp_key key ) - : simfil::Object(i, std::move(l), a, key) + : simfil::BaseObject(i, std::move(l), a, key) { } @@ -64,7 +64,7 @@ AttributeLayerList::newLayer(const std::string_view& name, size_t initialCapacit void AttributeLayerList::addLayer(const std::string_view& name, model_ptr l) { - addField(name, simfil::ModelNode::Ptr(std::move(l))); + addField(name, l); } bool AttributeLayerList::forEachLayer( @@ -78,7 +78,7 @@ bool AttributeLayerList::forEachLayer( log().warn("Don't add anything other than AttributeLayers into AttributeLayerLists!"); continue; } - auto attrLayer = static_cast(model()).resolveAttributeLayer(*value); + auto attrLayer = static_cast(model()).resolve(*value); if (!cb(*layerName, attrLayer)) return false; } diff --git a/libs/model/src/feature.cpp b/libs/model/src/feature.cpp index e4100914..24ffc8fd 100644 --- a/libs/model/src/feature.cpp +++ b/libs/model/src/feature.cpp @@ -23,12 +23,12 @@ Feature::Feature(Feature::Data& d, model_ptr Feature::id() const { - return model().resolveFeatureId(*Ptr::make(model_, data_->id_)); + return model().resolve(data_->id_); } std::string_view mapget::Feature::typeId() const { - return model().resolveFeatureId(*Ptr::make(model_, data_->id_))->typeId(); + return model().resolve(data_->id_)->typeId(); } model_ptr Feature::geom() @@ -46,7 +46,7 @@ model_ptr Feature::geomOrNull() const { if (!data_->geom_) return {}; - return model().resolveGeometryCollection(*Ptr::make(model_, data_->geom_)); + return model().resolve(data_->geom_); } model_ptr Feature::attributeLayers() @@ -64,7 +64,7 @@ model_ptr Feature::attributeLayersOrNull() const { if (!data_->attrLayers_) return {}; - return model().resolveAttributeLayerList(*Ptr::make(model_, data_->attrLayers_)); + return model().resolve(data_->attrLayers_); } model_ptr Feature::attributes() @@ -82,7 +82,7 @@ model_ptr Feature::attributesOrNull() const { if (!data_->attrs_) return {}; - return model().resolveObject(Ptr::make(model_, data_->attrs_)); + return model().resolve(data_->attrs_); } model_ptr Feature::relations() @@ -100,7 +100,7 @@ model_ptr Feature::relationsOrNull() const { if (!data_->relations_) return {}; - return model().resolveArray(Ptr::make(model_, data_->relations_)); + return model().resolve(data_->relations_); } tl::expected, simfil::Error> @@ -154,7 +154,7 @@ uint32_t Feature::size() const simfil::ModelNode::Ptr Feature::get(const simfil::StringId& f) const { if (f == StringPool::SourceDataStr) - return ModelNode::Ptr::make(model().shared_from_this(), data_->sourceData_); + return model().resolve(data_->sourceData_); for (auto const& [fieldName, fieldValue] : fields_) if (fieldName == f) @@ -194,7 +194,7 @@ void Feature::updateFields() { // Add id field fields_.emplace_back(StringPool::IdStr, Ptr::make(model_, data_->id_)); - auto idNode = model().resolveFeatureId(*fields_.back().second); + auto idNode = model().resolve(*fields_.back().second); // Add type id field fields_.emplace_back( @@ -297,7 +297,7 @@ uint32_t Feature::numRelations() const model_ptr Feature::getRelation(uint32_t index) const { if (data_->relations_) - return model().resolveRelation(*relationsOrNull()->at(index)); + return model().resolve(*relationsOrNull()->at(index)); return {}; } @@ -307,7 +307,7 @@ bool Feature::forEachRelation(std::function&)> co if (!relationsPtr || !callback) return true; for (auto const& relation : *relationsPtr) { - if (!callback(model().resolveRelation(*relation))) + if (!callback(model().resolve(*relation))) return false; } return true; @@ -349,7 +349,8 @@ Feature::filterRelations(const std::string_view& name) const model_ptr Feature::sourceDataReferences() const { if (data_->sourceData_) - return model().resolveSourceDataReferenceCollection(*model_ptr::make(model_, data_->sourceData_)); + return model().resolve( + *model_ptr::make(model_, data_->sourceData_)); return {}; } @@ -370,7 +371,7 @@ Feature::FeaturePropertyView::FeaturePropertyView( data_(&d) { if (data_->attrs_) - attrs_ = model().resolveObject(Ptr::make(model_, data_->attrs_)); + attrs_ = model().resolve(data_->attrs_); } simfil::ValueType Feature::FeaturePropertyView::type() const @@ -419,7 +420,8 @@ simfil::StringId Feature::FeaturePropertyView::keyAt(int64_t i) const bool Feature::FeaturePropertyView::iterate(const simfil::ModelNode::IterCallback& cb) const { if (data_->attrLayers_) { - if (!cb(*model().resolveAttributeLayerList(*Ptr::make(model_, data_->attrLayers_)))) + if (!cb(*model().resolve( + *Ptr::make(model_, data_->attrLayers_)))) return false; } if (attrs_) diff --git a/libs/model/src/featureid.cpp b/libs/model/src/featureid.cpp index 79cf6dc9..b8154038 100644 --- a/libs/model/src/featureid.cpp +++ b/libs/model/src/featureid.cpp @@ -12,7 +12,7 @@ FeatureId::FeatureId(FeatureId::Data& data, simfil::detail::mp_key key) : simfil::MandatoryDerivedModelNodeBase(l, a, key), data_(&data), - fields_(model().resolveObject(Ptr::make(l, data_->idParts_))) + fields_(model().resolve(data_->idParts_)) { } diff --git a/libs/model/src/featurelayer.cpp b/libs/model/src/featurelayer.cpp index 9402d1b5..885311cc 100644 --- a/libs/model/src/featurelayer.cpp +++ b/libs/model/src/featurelayer.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -116,7 +117,7 @@ namespace mapget { struct TileFeatureLayer::Impl { - simfil::ModelNodeAddress featureIdPrefix_; + ModelNodeAddress featureIdPrefix_; sfl::segmented_vector features_; sfl::segmented_vector attributes_; @@ -136,7 +137,7 @@ struct TileFeatureLayer::Impl { */ struct FeatureAddrWithIdHash { - simfil::ModelNodeAddress featureAddr_; + ModelNodeAddress featureAddr_; uint64_t idHash_ = 0; template @@ -314,16 +315,16 @@ simfil::model_ptr TileFeatureLayer::newFeature( auto featureIndex = impl_->features_.size(); impl_->features_.emplace_back(Feature::Data{ - simfil::ModelNodeAddress{ColumnId::FeatureIds, (uint32_t)featureIdIndex}, - simfil::ModelNodeAddress{Null, 0}, - simfil::ModelNodeAddress{Null, 0}, - simfil::ModelNodeAddress{Null, 0}, - simfil::ModelNodeAddress{Null, 0}, + ModelNodeAddress{ColumnId::FeatureIds, (uint32_t)featureIdIndex}, + ModelNodeAddress{Null, 0}, + ModelNodeAddress{Null, 0}, + ModelNodeAddress{Null, 0}, + ModelNodeAddress{Null, 0}, }); auto result = Feature( impl_->features_.back(), shared_from_this(), - simfil::ModelNodeAddress{ColumnId::Features, (uint32_t)featureIndex}, + ModelNodeAddress{ColumnId::Features, (uint32_t)featureIndex}, mpKey_); // Add feature hash index entry. @@ -336,7 +337,7 @@ simfil::model_ptr TileFeatureLayer::newFeature( // Note: Here we rely on the assertion that the root_ collection // contains only references to feature nodes, in the order // of the feature node column. - addRoot(simfil::ModelNode::Ptr(result)); + addRoot(ModelNode::Ptr(result)); setInfo("Size/Features", numRoots()); return result; } @@ -397,7 +398,7 @@ TileFeatureLayer::newRelation(const std::string_view& name, const model_ptr TileFeatureLayer::getIdPrefix() { if (impl_->featureIdPrefix_) - return resolveObject(simfil::ModelNode::Ptr::make(shared_from_this(), impl_->featureIdPrefix_)); + return resolve(impl_->featureIdPrefix_); return {}; } @@ -509,193 +510,212 @@ model_ptr TileFeatureLayer::newValidityCollection(size_t initialC mpKey_); } -model_ptr TileFeatureLayer::resolveAttributeLayer(simfil::ModelNode const& n) const +// Short aliases to keep resolve hook signatures compact. +using simfil::ModelNode; +using simfil::res::tag; + +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - if (n.addr().column() != ColumnId::AttributeLayers) + if (node.addr().column() != TileFeatureLayer::ColumnId::AttributeLayers) raise("Cannot cast this node to an AttributeLayer."); return AttributeLayer( - impl_->attrLayers_[n.addr().index()], - shared_from_this(), - n.addr(), - mpKey_); + model.impl_->attrLayers_[node.addr().index()], + model.shared_from_this(), + node.addr(), + model.mpKey_); } -model_ptr TileFeatureLayer::resolveAttributeLayerList(simfil::ModelNode const& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - if (n.addr().column() != ColumnId::AttributeLayerLists) + if (node.addr().column() != TileFeatureLayer::ColumnId::AttributeLayerLists) raise("Cannot cast this node to an AttributeLayerList."); return AttributeLayerList( - impl_->attrLayerLists_[n.addr().index()], - shared_from_this(), - n.addr(), - mpKey_); + model.impl_->attrLayerLists_[node.addr().index()], + model.shared_from_this(), + node.addr(), + model.mpKey_); } -model_ptr TileFeatureLayer::resolveAttribute(simfil::ModelNode const& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - if (n.addr().column() != ColumnId::Attributes) + if (node.addr().column() != TileFeatureLayer::ColumnId::Attributes) raise("Cannot cast this node to an Attribute."); return Attribute( - &impl_->attributes_[n.addr().index()], - shared_from_this(), - n.addr(), - mpKey_); + &model.impl_->attributes_[node.addr().index()], + model.shared_from_this(), + node.addr(), + model.mpKey_); } -model_ptr TileFeatureLayer::resolveFeature(simfil::ModelNode const& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - if (n.addr().column() != ColumnId::Features) + if (node.addr().column() != TileFeatureLayer::ColumnId::Features) raise("Cannot cast this node to a Feature."); return Feature( - impl_->features_[n.addr().index()], - shared_from_this(), - n.addr(), - mpKey_); + model.impl_->features_[node.addr().index()], + model.shared_from_this(), + node.addr(), + model.mpKey_); } -model_ptr TileFeatureLayer::resolveFeatureId(simfil::ModelNode const& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - if (n.addr().column() != ColumnId::FeatureIds) + if (node.addr().column() != TileFeatureLayer::ColumnId::FeatureIds) raise("Cannot cast this node to a FeatureId."); return FeatureId( - impl_->featureIds_[n.addr().index()], - shared_from_this(), - n.addr(), - mpKey_); + model.impl_->featureIds_[node.addr().index()], + model.shared_from_this(), + node.addr(), + model.mpKey_); } -model_ptr TileFeatureLayer::resolveRelation(const simfil::ModelNode& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - if (n.addr().column() != ColumnId::Relations) + if (node.addr().column() != TileFeatureLayer::ColumnId::Relations) raise("Cannot cast this node to a Relation."); return Relation( - &impl_->relations_[n.addr().index()], - shared_from_this(), - n.addr(), - mpKey_); + &model.impl_->relations_[node.addr().index()], + model.shared_from_this(), + node.addr(), + model.mpKey_); } -model_ptr TileFeatureLayer::resolvePoint(const simfil::ModelNode& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - if (n.addr().column() != ColumnId::Points) + switch (node.addr().column()) { + case TileFeatureLayer::ColumnId::Points: + return PointNode(node, &model.impl_->geom_.at(node.addr().index()), model.mpKey_); + case TileFeatureLayer::ColumnId::ValidityPoints: + return PointNode(node, &model.impl_->validities_.at(node.addr().index()), model.mpKey_); + default: raise("Cannot cast this node to a Point."); - return PointNode( - n, &impl_->geom_.at(n.addr().index()), mpKey_); + } } -model_ptr TileFeatureLayer::resolveValidityPoint(const simfil::ModelNode& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - if (n.addr().column() != ColumnId::ValidityPoints) - raise("Cannot cast this node to a ValidityPoint."); - return PointNode( - n, &impl_->validities_.at(n.addr().index()), mpKey_); + return PointBufferNode( + &model.impl_->geom_.at(node.addr().index()), + model.shared_from_this(), + node.addr(), + model.mpKey_); } -model_ptr TileFeatureLayer::resolveValidity(simfil::ModelNode const& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - if (n.addr().column() != ColumnId::Validities) - raise("Cannot cast this node to a Validity."); - return Validity( - &impl_->validities_[n.addr().index()], - shared_from_this(), - n.addr(), - mpKey_); + auto* geomData = &model.impl_->geom_.at(node.addr().index()); + using MutableGeomData = + std::remove_const_t>; + return Geometry( + const_cast(geomData), // FIXME: const_cast?! + model.shared_from_this(), + node.addr(), + model.mpKey_); } -model_ptr TileFeatureLayer::resolveValidityCollection(const simfil::ModelNode& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - if (n.addr().column() != ColumnId::ValidityCollections) - raise("Cannot cast this node to a ValidityCollection."); - return MultiValidity( - shared_from_this(), - n.addr(), - mpKey_); + return GeometryCollection( + model.shared_from_this(), node.addr(), model.mpKey_); } -model_ptr TileFeatureLayer::resolvePointBuffer(const simfil::ModelNode& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - return PointBufferNode( - &impl_->geom_.at(n.addr().index()), - shared_from_this(), - n.addr(), - mpKey_); + return MeshNode( + &model.impl_->geom_.at(node.addr().index()), + model.shared_from_this(), + node.addr(), + model.mpKey_); } -model_ptr TileFeatureLayer::resolvePolygon(const simfil::ModelNode& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - return PolygonNode( - shared_from_this(), - n.addr(), - mpKey_); + return MeshTriangleCollectionNode(node, model.mpKey_); } -model_ptr TileFeatureLayer::resolveMesh(const simfil::ModelNode& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - return MeshNode( - &impl_->geom_.at(n.addr().index()), - shared_from_this(), - n.addr(), - mpKey_); + switch (node.addr().column()) { + case TileFeatureLayer::ColumnId::LinearRing: + return LinearRingNode(node, model.mpKey_); + case TileFeatureLayer::ColumnId::MeshTriangleLinearRing: + return LinearRingNode(node, 3, model.mpKey_); + default: + raise("Cannot cast this node to a LinearRing."); + } } -model_ptr TileFeatureLayer::resolveLinearRing(const simfil::ModelNode& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - return LinearRingNode(n, mpKey_); + return PolygonNode( + model.shared_from_this(), + node.addr(), + model.mpKey_); } -model_ptr TileFeatureLayer::resolveGeometry(const simfil::ModelNode& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - return Geometry( - &const_cast(impl_->geom_.at(n.addr().index())), // FIXME: const_cast?! - shared_from_this(), - n.addr(), - mpKey_); -} + if (node.addr().column() != TileFeatureLayer::ColumnId::SourceDataReferenceCollections) + raise("Cannot cast this node to an SourceDataReferenceCollection."); -model_ptr TileFeatureLayer::resolveMeshTriangleLinearRing(const simfil::ModelNode& n) const -{ - return LinearRingNode(n, 3, mpKey_); + auto [index, size] = modelAddressToSourceDataAddressList(node.addr().index()); + return SourceDataReferenceCollection(index, size, model.shared_from_this(), node.addr(), model.mpKey_); } -model_ptr TileFeatureLayer::resolveMeshTriangleCollection(const simfil::ModelNode& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - return MeshTriangleCollectionNode(n, mpKey_); -} + if (node.addr().column() != TileFeatureLayer::ColumnId::SourceDataReferences) + raise("Cannot cast this node to an SourceDataReferenceItem."); -model_ptr -TileFeatureLayer::resolveGeometryCollection(const simfil::ModelNode& n) const -{ - return GeometryCollection( - shared_from_this(), n.addr(), mpKey_); + const auto* data = &model.impl_->sourceDataReferences_.at(node.addr().index()); + return SourceDataReferenceItem(data, model.shared_from_this(), node.addr(), model.mpKey_); } -model_ptr -TileFeatureLayer::resolveSourceDataReferenceCollection(const simfil::ModelNode& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - if (n.addr().column() != ColumnId::SourceDataReferenceCollections) - raise("Cannot cast this node to an SourceDataReferenceCollection."); - - auto [index, size] = modelAddressToSourceDataAddressList(n.addr().index()); - const auto& data = impl_->sourceDataReferences_; - return SourceDataReferenceCollection(index, size, shared_from_this(), n.addr(), mpKey_); + if (node.addr().column() != TileFeatureLayer::ColumnId::Validities) + raise("Cannot cast this node to a Validity."); + return Validity( + &model.impl_->validities_[node.addr().index()], + model.shared_from_this(), + node.addr(), + model.mpKey_); } -model_ptr -TileFeatureLayer::resolveSourceDataReferenceItem(const simfil::ModelNode& n) const +template<> +model_ptr resolveInternal(tag, TileFeatureLayer const& model, ModelNode const& node) { - if (n.addr().column() != ColumnId::SourceDataReferences) - raise("Cannot cast this node to an SourceDataReferenceItem."); - - const auto* data = &impl_->sourceDataReferences_.at(n.addr().index()); - return SourceDataReferenceItem(data, shared_from_this(), n.addr(), mpKey_); + if (node.addr().column() != TileFeatureLayer::ColumnId::ValidityCollections) + raise("Cannot cast this node to a ValidityCollection."); + return MultiValidity( + model.shared_from_this(), + node.addr(), + model.mpKey_); } -tl::expected TileFeatureLayer::resolve(const simfil::ModelNode& n, const simfil::Model::ResolveFn& cb) const +tl::expected TileFeatureLayer::resolve(const ModelNode& n, const simfil::Model::ResolveFn& cb) const { switch (n.addr().column()) { case ColumnId::Features: - cb(*resolveFeature(n)); + cb(*resolve(n)); return {}; case ColumnId::FeatureProperties: cb(Feature::FeaturePropertyView( @@ -706,61 +726,61 @@ tl::expected TileFeatureLayer::resolve(const simfil::ModelN )); return {}; case ColumnId::FeatureIds: - cb(*resolveFeatureId(n)); + cb(*resolve(n)); return {}; case ColumnId::Attributes: - cb(*resolveAttribute(n)); + cb(*resolve(n)); return {}; case ColumnId::AttributeLayers: - cb(*resolveAttributeLayer(n)); + cb(*resolve(n)); return {}; case ColumnId::AttributeLayerLists: - cb(*resolveAttributeLayerList(n)); + cb(*resolve(n)); return {}; case ColumnId::Relations: - cb(*resolveRelation(n)); + cb(*resolve(n)); return {}; case ColumnId::Points: - cb(*resolvePoint(n)); + cb(*resolve(n)); return {}; case ColumnId::PointBuffers: - cb(*resolvePointBuffer(n)); + cb(*resolve(n)); return {}; case ColumnId::Geometries: - cb(*resolveGeometry(n)); + cb(*resolve(n)); return {}; case ColumnId::GeometryCollections: - cb(*resolveGeometryCollection(n)); + cb(*resolve(n)); return {}; case ColumnId::Polygon: - cb(*resolvePolygon(n)); + cb(*resolve(n)); return {}; case ColumnId::Mesh: - cb(*resolveMesh(n)); + cb(*resolve(n)); return {}; case ColumnId::MeshTriangleCollection: - cb(*resolveMeshTriangleCollection(n)); + cb(*resolve(n)); return {}; case ColumnId::MeshTriangleLinearRing: - cb(*resolveMeshTriangleLinearRing(n)); + cb(*resolve(n)); return {}; case ColumnId::LinearRing: - cb(*resolveLinearRing(n)); + cb(*resolve(n)); return {}; case ColumnId::SourceDataReferenceCollections: - cb(*resolveSourceDataReferenceCollection(n)); + cb(*resolve(n)); return {}; case ColumnId::SourceDataReferences: - cb(*resolveSourceDataReferenceItem(n)); + cb(*resolve(n)); return {}; case ColumnId::Validities: - cb(*resolveValidity(n)); + cb(*resolve(n)); return {}; case ColumnId::ValidityPoints: - cb(*resolveValidityPoint(n)); + cb(*resolve(n)); return {}; case ColumnId::ValidityCollections: - cb(*resolveValidityCollection(n)); + cb(*resolve(n)); return {}; } @@ -974,7 +994,7 @@ model_ptr TileFeatureLayer::at(size_t i) const auto rootResult = root(i); if (!rootResult) return {}; - return resolveFeature(**rootResult); + return resolve(**rootResult); } model_ptr @@ -994,7 +1014,7 @@ TileFeatureLayer::find(const std::string_view& type, const KeyValueViewPairs& qu // Iterate through potential matches to handle hash collisions. while (it != impl_->featureHashIndex_.end() && it->idHash_ == hash) { - auto feature = resolveFeature(*simfil::ModelNode::Ptr::make(shared_from_this(), it->featureAddr_)); + auto feature = resolve(it->featureAddr_); if (feature->id()->typeId() == type) { auto featureIdParts = stripOptionalIdParts(feature->id()->keyValuePairs(), primaryIdComposition); // Ensure that ID parts match exactly, not just the hash. @@ -1090,10 +1110,10 @@ TileFeatureLayer::setStrings(std::shared_ptr const& newDict) return {}; } -simfil::ModelNode::Ptr TileFeatureLayer::clone( - std::unordered_map& cache, +ModelNode::Ptr TileFeatureLayer::clone( + std::unordered_map& cache, const TileFeatureLayer::Ptr& otherLayer, - const simfil::ModelNode::Ptr& otherNode) + const ModelNode::Ptr& otherNode) { auto it = cache.find(otherNode->addr().value_); if (it != cache.end()) { @@ -1104,7 +1124,7 @@ simfil::ModelNode::Ptr TileFeatureLayer::clone( ModelNode::Ptr& newCacheNode = cache[otherNode->addr().value_]; switch (otherNode->addr().column()) { case Objects: { - auto resolved = otherLayer->resolveObject(otherNode); + auto resolved = otherLayer->resolve(otherNode); auto newNode = newObject(resolved->size()); newCacheNode = newNode; for (auto [key, value] : resolved->fields()) { @@ -1115,7 +1135,7 @@ simfil::ModelNode::Ptr TileFeatureLayer::clone( break; } case Arrays: { - auto resolved = otherLayer->resolveArray(otherNode); + auto resolved = otherLayer->resolve(otherNode); auto newNode = newArray(resolved->size()); newCacheNode = newNode; for (auto value : *resolved) { @@ -1127,7 +1147,7 @@ simfil::ModelNode::Ptr TileFeatureLayer::clone( // TODO: This implementation is not great, because it does not respect // Geometry views - it just converts every Geometry to a self-contained one. // TODO: Clone geometry name. - auto resolved = otherLayer->resolveGeometry(*otherNode); + auto resolved = otherLayer->resolve(*otherNode); auto newNode = newGeometry(resolved->geomType(), resolved->numPoints()); newCacheNode = newNode; resolved->forEachPoint( @@ -1139,13 +1159,13 @@ simfil::ModelNode::Ptr TileFeatureLayer::clone( break; } case ColumnId::GeometryCollections: { - auto resolved = otherLayer->resolveGeometryCollection(*otherNode); + auto resolved = otherLayer->resolve(*otherNode); auto newNode = newGeometryCollection(resolved->numGeometries()); newCacheNode = newNode; resolved->forEachGeometry( [this, &newNode, &cache, &otherLayer](auto&& geom) { - newNode->addGeometry(resolveGeometry(*clone(cache, otherLayer, geom))); + newNode->addGeometry(resolve(*clone(cache, otherLayer, geom))); return true; }); break; @@ -1179,18 +1199,18 @@ simfil::ModelNode::Ptr TileFeatureLayer::clone( raise("Cannot clone entire feature yet."); } case ColumnId::FeatureIds: { - auto resolved = otherLayer->resolveFeatureId(*otherNode); + auto resolved = otherLayer->resolve(*otherNode); auto newNode = newFeatureId(resolved->typeId(), resolved->keyValuePairs()); newCacheNode = newNode; break; } case ColumnId::Attributes: { - auto resolved = otherLayer->resolveAttribute(*otherNode); + auto resolved = otherLayer->resolve(*otherNode); auto newNode = newAttribute(resolved->name()); newCacheNode = newNode; if (resolved->validityOrNull()) { newNode->setValidity( - resolveValidityCollection(*clone(cache, otherLayer, resolved->validityOrNull()))); + resolve(*clone(cache, otherLayer, resolved->validityOrNull()))); } resolved->forEachField( [this, &newNode, &cache, &otherLayer](auto&& key, auto&& value) @@ -1201,7 +1221,7 @@ simfil::ModelNode::Ptr TileFeatureLayer::clone( break; } case ColumnId::Validities: { - auto resolved = otherLayer->resolveValidity(*otherNode); + auto resolved = otherLayer->resolve(*otherNode); auto newNode = newValidity(); newCacheNode = newNode; newNode->setDirection(resolved->direction()); @@ -1209,7 +1229,8 @@ simfil::ModelNode::Ptr TileFeatureLayer::clone( case Validity::NoGeometry: break; case Validity::SimpleGeometry: - newNode->setSimpleGeometry(resolveGeometry(*clone(cache, otherLayer, resolved->simpleGeometry()))); + newNode->setSimpleGeometry(resolve( + *clone(cache, otherLayer, resolved->simpleGeometry()))); break; case Validity::OffsetPointValidity: if (resolved->geometryOffsetType() == Validity::GeoPosOffset) { @@ -1231,54 +1252,56 @@ simfil::ModelNode::Ptr TileFeatureLayer::clone( break; } case ColumnId::ValidityCollections: { - auto resolved = otherLayer->resolveValidityCollection(*otherNode); + auto resolved = otherLayer->resolve(*otherNode); auto newNode = newValidityCollection(resolved->size()); newCacheNode = newNode; for (auto value : *resolved) { - newNode->append(resolveValidity(*clone(cache, otherLayer, value))); + newNode->append(resolve(*clone(cache, otherLayer, value))); } break; } case ColumnId::AttributeLayers: { - auto resolved = otherLayer->resolveAttributeLayer(*otherNode); + auto resolved = otherLayer->resolve(*otherNode); auto newNode = newAttributeLayer(resolved->size()); newCacheNode = newNode; for (auto [key, value] : resolved->fields()) { if (auto keyStr = otherLayer->strings()->resolve(key)) { - newNode->addField(*keyStr, clone(cache, otherLayer, value)); + auto cloned = clone(cache, otherLayer, value); + newNode->addField(*keyStr, resolve(*cloned)); } } break; } case ColumnId::AttributeLayerLists: { - auto resolved = otherLayer->resolveAttributeLayerList(*otherNode); + auto resolved = otherLayer->resolve(*otherNode); auto newNode = newAttributeLayers(resolved->size()); newCacheNode = newNode; for (auto [key, value] : resolved->fields()) { if (auto keyStr = otherLayer->strings()->resolve(key)) { - newNode->addField(*keyStr, clone(cache, otherLayer, value)); + auto cloned = clone(cache, otherLayer, value); + newNode->addField(*keyStr, resolve(*cloned)); } } break; } case ColumnId::Relations: { - auto resolved = otherLayer->resolveRelation(*otherNode); + auto resolved = otherLayer->resolve(*otherNode); auto newNode = newRelation( resolved->name(), - resolveFeatureId(*clone(cache, otherLayer, resolved->target()))); + resolve(*clone(cache, otherLayer, resolved->target()))); if (resolved->sourceValidityOrNull()) { - newNode->setSourceValidity(resolveValidityCollection( + newNode->setSourceValidity(resolve( *clone(cache, otherLayer, resolved->sourceValidityOrNull()))); } if (resolved->targetValidityOrNull()) { - newNode->setTargetValidity(resolveValidityCollection( + newNode->setTargetValidity(resolve( *clone(cache, otherLayer, resolved->targetValidityOrNull()))); } newCacheNode = newNode; break; } case ColumnId::SourceDataReferenceCollections: { - auto resolved = otherLayer->resolveSourceDataReferenceCollection(*otherNode); + auto resolved = otherLayer->resolve(*otherNode); auto items = std::vector( otherLayer->impl_->sourceDataReferences_.begin() + resolved->offset_, otherLayer->impl_->sourceDataReferences_.begin() + resolved->offset_ + resolved->size_); @@ -1296,7 +1319,7 @@ simfil::ModelNode::Ptr TileFeatureLayer::clone( case ColumnId::ValidityPoints: raiseFmt("Encountered unexpected column type {} in clone().", otherNode->addr().column()); default: { - newCacheNode = ModelNode::Ptr::make(shared_from_this(), otherNode->addr()); + newCacheNode = resolve(otherNode->addr()); } } cache.insert({otherNode->addr().value_, newCacheNode}); @@ -1304,7 +1327,7 @@ simfil::ModelNode::Ptr TileFeatureLayer::clone( } void TileFeatureLayer::clone( - std::unordered_map& clonedModelNodes, + std::unordered_map& clonedModelNodes, const TileFeatureLayer::Ptr& otherLayer, const Feature& otherFeature, const std::string_view& type, @@ -1321,7 +1344,7 @@ void TileFeatureLayer::clone( } auto lookupOrClone = - [&](simfil::ModelNode::Ptr const& n) -> simfil::ModelNode::Ptr + [&](ModelNode::Ptr const& n) -> ModelNode::Ptr { return clone(clonedModelNodes, otherLayer, n); }; @@ -1341,7 +1364,7 @@ void TileFeatureLayer::clone( auto baseAttrLayers = cloneTarget->attributeLayers(); for (auto const& [key, value] : attrLayers->fields()) { if (auto keyStr = otherLayer->strings()->resolve(key)) { - baseAttrLayers->addField(*keyStr, lookupOrClone(value)); + baseAttrLayers->addField(*keyStr, resolve(*lookupOrClone(value))); } } } @@ -1353,7 +1376,7 @@ void TileFeatureLayer::clone( [this, &baseGeom, &lookupOrClone](auto&& geomElement) { baseGeom->addGeometry( - resolveGeometry(*lookupOrClone(geomElement))); + resolve(*lookupOrClone(geomElement))); return true; }); } @@ -1363,7 +1386,7 @@ void TileFeatureLayer::clone( otherFeature.forEachRelation( [this, &cloneTarget, &lookupOrClone](auto&& rel) { - auto newRel = resolveRelation(*lookupOrClone(rel)); + auto newRel = resolve(*lookupOrClone(rel)); cloneTarget->addRelation(newRel); return true; }); diff --git a/libs/model/src/geometry.cpp b/libs/model/src/geometry.cpp index bf6d5444..53fb65bd 100644 --- a/libs/model/src/geometry.cpp +++ b/libs/model/src/geometry.cpp @@ -74,7 +74,7 @@ ModelNode::Ptr GeometryCollection::at(int64_t i) const { if (auto singleGeomEntry = singleGeom()) return singleGeomEntry->at(i); if (i == 0) return model_ptr::make(GeometryCollectionStr, model_); - if (i == 1) return ModelNode::Ptr::make(model_, ModelNodeAddress{simfil::ModelPool::Arrays, addr_.index()}); + if (i == 1) return model().resolve(ModelNodeAddress{simfil::ModelPool::Arrays, addr_.index()}); throw std::out_of_range("geom collection: Out of range."); } @@ -103,8 +103,8 @@ StringId GeometryCollection::keyAt(int64_t i) const { model_ptr GeometryCollection::newGeometry(GeomType type, size_t initialCapacity) { auto result = model().newGeometry(type, initialCapacity); - auto arrayPtr = ModelNode::Ptr::make(model_, ModelNodeAddress{simfil::ModelPool::Arrays, addr_.index()}); - model().resolveArray(arrayPtr)->append(result); + auto array = model().resolve(ModelNodeAddress{simfil::ModelPool::Arrays, addr_.index()}); + array->append(result); return result; } @@ -120,16 +120,16 @@ bool GeometryCollection::iterate(const IterCallback& cb) const ModelNode::Ptr GeometryCollection::singleGeom() const { if (model().arrayMemberStorage().size((ArrayIndex)addr_.index()) == 1) { - auto arrayPtr = ModelNode::Ptr::make(model_, ModelNodeAddress{simfil::ModelPool::Arrays, addr_.index()}); - return model().resolveArray(arrayPtr)->at(0); + auto array = model().resolve(ModelNodeAddress{simfil::ModelPool::Arrays, addr_.index()}); + return array->at(0); } return {}; } void GeometryCollection::addGeometry(const model_ptr& geom) { - auto arrayPtr = ModelNode::Ptr::make(model_, ModelNodeAddress{simfil::ModelPool::Arrays, addr_.index()}); - model().resolveArray(arrayPtr)->append(ModelNode::Ptr(geom)); + auto array = model().resolve(ModelNodeAddress{simfil::ModelPool::Arrays, addr_.index()}); + array->append(ModelNode::Ptr(geom)); } size_t GeometryCollection::numGeometries() const @@ -194,7 +194,7 @@ uint32_t Geometry::size() const { ModelNode::Ptr Geometry::get(const StringId& f) const { if (f == StringPool::SourceDataStr && geomData_->sourceDataReferences_) { - return ModelNode::Ptr::make(model_, geomData_->sourceDataReferences_); + return model().resolve(geomData_->sourceDataReferences_); } if (f == StringPool::TypeStr) { return model_ptr::make( @@ -207,14 +207,14 @@ ModelNode::Ptr Geometry::get(const StringId& f) const { if (f == StringPool::CoordinatesStr) { switch (geomData_->type_) { case GeomType::Polygon: - return ModelNode::Ptr::make( - model_, ModelNodeAddress{TileFeatureLayer::ColumnId::Polygon, addr_.index()}); + return model().resolve( + ModelNodeAddress{TileFeatureLayer::ColumnId::Polygon, addr_.index()}); case GeomType::Mesh: - return ModelNode::Ptr::make( - model_, ModelNodeAddress{TileFeatureLayer::ColumnId::Mesh, addr_.index()}); + return model().resolve( + ModelNodeAddress{TileFeatureLayer::ColumnId::Mesh, addr_.index()}); default: - return ModelNode::Ptr::make( - model_, ModelNodeAddress{TileFeatureLayer::ColumnId::PointBuffers, addr_.index()}); + return model().resolve( + ModelNodeAddress{TileFeatureLayer::ColumnId::PointBuffers, addr_.index()}); } } if (f == StringPool::NameStr) { @@ -243,7 +243,7 @@ StringId Geometry::keyAt(int64_t i) const { model_ptr Geometry::sourceDataReferences() const { if (geomData_->sourceDataReferences_) - return model().resolveSourceDataReferenceCollection(*model_ptr::make(model_, geomData_->sourceDataReferences_)); + return model().resolve(geomData_->sourceDataReferences_); return {}; } @@ -489,8 +489,8 @@ ModelNode::Ptr PolygonNode::at(int64_t index) const { // Index 0 is the outer ring, all following rings are holes if (index == 0) - return ModelNode::Ptr::make( - model_, ModelNodeAddress{TileFeatureLayer::ColumnId::LinearRing, addr_.index()}); + return model().resolve( + ModelNodeAddress{TileFeatureLayer::ColumnId::LinearRing, addr_.index()}); throw std::out_of_range("PolygonNode: index out of bounds."); } @@ -539,8 +539,9 @@ ValueType MeshNode::type() const ModelNode::Ptr MeshNode::at(int64_t index) const { if (0 <= index && index < size_) - return ModelNode::Ptr::make( - model_, ModelNodeAddress{TileFeatureLayer::ColumnId::MeshTriangleCollection, addr_.index()}, index); + return model().resolve( + ModelNodeAddress{TileFeatureLayer::ColumnId::MeshTriangleCollection, addr_.index()}, + index); throw std::out_of_range("MeshNode: index out of bounds."); } @@ -571,7 +572,9 @@ ValueType MeshTriangleCollectionNode::type() const ModelNode::Ptr MeshTriangleCollectionNode::at(int64_t index) const { if (index == 0) - return ModelNode::Ptr::make(model_, ModelNodeAddress{TileFeatureLayer::ColumnId::MeshTriangleLinearRing, addr_.index()}, index_); + return model().resolve( + ModelNodeAddress{TileFeatureLayer::ColumnId::MeshTriangleLinearRing, addr_.index()}, + index_); throw std::out_of_range("MeshTriangleCollectionNode: index out of bounds."); } @@ -698,9 +701,8 @@ uint32_t LinearRingNode::size() const model_ptr LinearRingNode::vertexBuffer() const { - auto ptr = ModelNode::Ptr::make( - model_, ModelNodeAddress{TileFeatureLayer::ColumnId::PointBuffers, addr_.index()}, 0); - return model().resolvePointBuffer(*ptr); + return model().resolve( + ModelNodeAddress{TileFeatureLayer::ColumnId::PointBuffers, addr_.index()}); } /** ModelNode impls. for VertexBufferNode */ @@ -723,8 +725,8 @@ PointBufferNode::PointBufferNode(Geometry::Data const* geomData, while (baseGeomData_->isView_) { offset_ += baseGeomData_->detail_.view_.offset_; baseGeomAddress_ = baseGeomData_->detail_.view_.baseGeometry_; - baseGeomData_ = model().resolveGeometry( - *ModelNode::Ptr::make(model_, baseGeomData_->detail_.view_.baseGeometry_))->geomData_; + baseGeomData_ = model().resolve( + baseGeomData_->detail_.view_.baseGeometry_)->geomData_; } auto maxSize = 1 + storage_->size(baseGeomData_->detail_.geom_.vertexArray_); @@ -746,7 +748,9 @@ ModelNode::Ptr PointBufferNode::at(int64_t i) const { if (i < 0 || i >= size()) throw std::out_of_range("vertex-buffer: Out of range."); i += offset_; - return ModelNode::Ptr::make(model_, ModelNodeAddress{TileFeatureLayer::ColumnId::Points, baseGeomAddress_.index()}, i); + return model().resolve( + ModelNodeAddress{TileFeatureLayer::ColumnId::Points, baseGeomAddress_.index()}, + i); } uint32_t PointBufferNode::size() const { @@ -768,8 +772,9 @@ bool PointBufferNode::iterate(const IterCallback& cb) const cont = cb(node); }); for (auto i = 0u; i < size_; ++i) { - resolveAndCb(*ModelNode::Ptr::make( - model_, ModelNodeAddress{TileFeatureLayer::ColumnId::Points, baseGeomAddress_.index()}, (int64_t)i+offset_)); + resolveAndCb(*model().resolve( + ModelNodeAddress{TileFeatureLayer::ColumnId::Points, baseGeomAddress_.index()}, + (int64_t)i + offset_)); if (!cont) break; } diff --git a/libs/model/src/relation.cpp b/libs/model/src/relation.cpp index a0319328..85474881 100644 --- a/libs/model/src/relation.cpp +++ b/libs/model/src/relation.cpp @@ -22,25 +22,25 @@ Relation::Relation(Relation::Data* data, fields_.emplace_back( StringPool::TargetStr, [](Relation const& self) { - return ModelNode::Ptr::make(self.model().shared_from_this(), self.data_->targetFeatureId_); + return self.model().resolve(self.data_->targetFeatureId_); }); if (data_->sourceValidity_) fields_.emplace_back( StringPool::SourceValidityStr, [](Relation const& self) { - return ModelNode::Ptr::make(self.model().shared_from_this(), self.data_->sourceValidity_); + return self.model().resolve(self.data_->sourceValidity_); }); if (data_->targetValidity_) fields_.emplace_back( StringPool::TargetValidityStr, [](Relation const& self) { - return ModelNode::Ptr::make(self.model().shared_from_this(), self.data_->targetValidity_); + return self.model().resolve(self.data_->targetValidity_); }); if (data_->sourceData_) fields_.emplace_back( StringPool::SourceDataStr, [](Relation const& self) { - return ModelNode::Ptr::make(self.model().shared_from_this(), self.data_->sourceData_); + return self.model().resolve(self.data_->sourceData_); }); } @@ -58,7 +58,8 @@ model_ptr Relation::sourceValidityOrNull() const { if (!data_->sourceValidity_) return {}; - return model().resolveValidityCollection(*model_ptr::make(model_, data_->sourceValidity_)); + return model().resolve( + *model_ptr::make(model_, data_->sourceValidity_)); } void Relation::setSourceValidity(const model_ptr& validityGeom) @@ -81,7 +82,8 @@ model_ptr Relation::targetValidityOrNull() const { if (!data_->targetValidity_) return {}; - return model().resolveValidityCollection(*model_ptr::make(model_, data_->targetValidity_)); + return model().resolve( + *model_ptr::make(model_, data_->targetValidity_)); } void Relation::setTargetValidity(const model_ptr& validityGeom) @@ -98,13 +100,15 @@ std::string_view Relation::name() const model_ptr Relation::target() const { - return model().resolveFeatureId(*model_ptr::make(model_, data_->targetFeatureId_)); + return model().resolve( + *model_ptr::make(model_, data_->targetFeatureId_)); } model_ptr Relation::sourceDataReferences() const { if (data_->sourceData_) - return model().resolveSourceDataReferenceCollection(*model_ptr::make(model_, data_->sourceData_)); + return model().resolve( + *model_ptr::make(model_, data_->sourceData_)); return {}; } diff --git a/libs/model/src/sourcedata.cpp b/libs/model/src/sourcedata.cpp index a25af31b..ebf33471 100644 --- a/libs/model/src/sourcedata.cpp +++ b/libs/model/src/sourcedata.cpp @@ -50,7 +50,7 @@ simfil::model_ptr SourceDataCompoundNode::object() const { if (!data_->object_) return {}; - return model().resolveObject(ModelNode::Ptr::make(model_, data_->object_)); + return model().resolve(data_->object_); } ValueType SourceDataCompoundNode::type() const diff --git a/libs/model/src/sourcedatalayer.cpp b/libs/model/src/sourcedatalayer.cpp index 397a39f4..33e45541 100644 --- a/libs/model/src/sourcedatalayer.cpp +++ b/libs/model/src/sourcedatalayer.cpp @@ -100,22 +100,27 @@ model_ptr TileSourceDataLayer::newCompound(size_t initia mpKey_); } -model_ptr TileSourceDataLayer::resolveCompound(simfil::ModelNode const& n) const +// Short aliases to keep resolve hook signatures compact. +using simfil::ModelNode; +using simfil::res::tag; + +template<> +model_ptr resolveInternal(tag, TileSourceDataLayer const& model, ModelNode const& node) { - assert(n.addr().column() == Compound && "Unexpected column type!"); + assert(node.addr().column() == TileSourceDataLayer::Compound && "Unexpected column type!"); - auto& data = impl_->compounds_.at(n.addr().index()); + auto& data = model.impl_->compounds_.at(node.addr().index()); return SourceDataCompoundNode( &data, - std::static_pointer_cast(shared_from_this()), - n.addr(), - mpKey_); + std::static_pointer_cast(model.shared_from_this()), + node.addr(), + model.mpKey_); } tl::expected TileSourceDataLayer::resolve(const simfil::ModelNode& n, const ResolveFn& cb) const { if (n.addr().column() == Compound) { - cb(*resolveCompound(n)); + cb(*resolve(n)); return {}; } return ModelPool::resolve(n, cb); diff --git a/libs/model/src/sourcedatareference.cpp b/libs/model/src/sourcedatareference.cpp index 17e002ea..61113d59 100644 --- a/libs/model/src/sourcedatareference.cpp +++ b/libs/model/src/sourcedatareference.cpp @@ -20,8 +20,8 @@ ModelNode::Ptr SourceDataReferenceCollection::at(int64_t index) const if (index < 0 || index >= size_ || (offset_ + index) > 0xffffff) throw std::out_of_range("Index out of range"); - return ModelNode::Ptr::make( - model_, ModelNodeAddress{TileFeatureLayer::ColumnId::SourceDataReferences, static_cast(offset_ + index)}); + return model().resolve( + ModelNodeAddress{TileFeatureLayer::ColumnId::SourceDataReferences, static_cast(offset_ + index)}); } uint32_t SourceDataReferenceCollection::size() const @@ -41,7 +41,7 @@ void SourceDataReferenceCollection::forEachReference(std::function(*at(i))); } } diff --git a/libs/model/src/validity.cpp b/libs/model/src/validity.cpp index 0fd63a23..3f18445a 100644 --- a/libs/model/src/validity.cpp +++ b/libs/model/src/validity.cpp @@ -27,7 +27,7 @@ model_ptr Validity::featureId() const if (!data_->featureAddress_) { return {}; } - return model().resolveFeatureId(*Ptr::make(model_, data_->featureAddress_)); + return model().resolve(data_->featureAddress_); } void Validity::setFeatureId(model_ptr featureId) @@ -61,8 +61,7 @@ Validity::Validity(Validity::Data* data, StringPool::GeometryStr, [](Validity const& self) { - return ModelNode::Ptr::make( - self.model_, + return self.model().resolve( std::get(self.data_->geomDescr_)); }); return; @@ -109,8 +108,7 @@ Validity::Validity(Validity::Data* data, switch (self.data_->geomOffsetType_) { case InvalidOffsetType: return ModelNode::Ptr{}; case GeoPosOffset: - return ModelNode::Ptr::make( - self.model_, + return self.model().resolve( ModelNodeAddress{ TileFeatureLayer::ColumnId::ValidityPoints, self.addr().index()}, @@ -241,7 +239,7 @@ model_ptr Validity::simpleGeometry() const if (data_->geomDescrType_ != SimpleGeometry) { return {}; } - return model().resolveGeometry(*ModelNode::Ptr::make(model_, std::get(data_->geomDescr_))); + return model().resolve(std::get(data_->geomDescr_)); } SelfContainedGeometry Validity::computeGeometry( From 2db9691c0b21b76ca17cc7362e34ec1f809f41d0 Mon Sep 17 00:00:00 2001 From: Wagram Airiian Date: Tue, 10 Feb 2026 08:05:04 +0100 Subject: [PATCH 29/38] Fix output regarding ByteArray. --- libs/model/include/mapget/model/featurelayer.h | 2 +- libs/model/src/featureid.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/model/include/mapget/model/featurelayer.h b/libs/model/include/mapget/model/featurelayer.h index 07888c60..9f4ffbc6 100644 --- a/libs/model/include/mapget/model/featurelayer.h +++ b/libs/model/include/mapget/model/featurelayer.h @@ -233,7 +233,7 @@ class TileFeatureLayer : public TileLayer, public simfil::ModelPool * @param query Simfil query * @param node Model root node to query * @param anyMode Auto-wrap expression in `any(...)` - * @param autoWildcard Auto expand constant expressions to `** = ` + * @param autoWildcard Auto expand constant expressions to `** == ` */ struct QueryResult { // The list of values resulting from the query evaluation. diff --git a/libs/model/src/featureid.cpp b/libs/model/src/featureid.cpp index cb8293d5..228c3854 100644 --- a/libs/model/src/featureid.cpp +++ b/libs/model/src/featureid.cpp @@ -30,7 +30,7 @@ std::string FeatureId::toString() const auto addIdPart = [&result](auto&& v) { if constexpr (std::is_same_v, simfil::ByteArray>) { - raiseFmt("FeatureId part value '{}' cannot be a ByteArray.", v.toDisplayString()); + raiseFmt("FeatureId part value 'b\"{}\"' cannot be a ByteArray.", v.toHex()); } else if constexpr (!std::is_same_v, std::monostate>) { result << "." << v; } From 99c55266cca4285599133366fe63caf915827473 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Tue, 10 Feb 2026 12:36:36 +0100 Subject: [PATCH 30/38] Fix py-layer.h --- libs/pymapget/binding/py-layer.h | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/libs/pymapget/binding/py-layer.h b/libs/pymapget/binding/py-layer.h index bc641193..8b149852 100644 --- a/libs/pymapget/binding/py-layer.h +++ b/libs/pymapget/binding/py-layer.h @@ -96,8 +96,17 @@ void bindTileLayer(py::module_& m) std::visit( [&](auto&& vv) { - if constexpr (!std::is_same_v, std::monostate>) + using V = std::decay_t; + if constexpr (std::is_same_v) { + return; + } + else if constexpr (std::is_same_v) { + // Store bytes in hex to keep JSON valid and readable. + self.setInfo(k, vv.toHex()); + } + else { self.setInfo(k, vv); + } }, v); }, @@ -106,7 +115,7 @@ void bindTileLayer(py::module_& m) R"pbdoc( Set a JSON field to store sizes, construction times, and other arbitrary meta-information. The value may be - bool, int, double or string. + bool, int, double or string. ByteArray values are stored as hex strings. )pbdoc") .def( "set_prefix", From da32794ed15d25fb1745ed4ff24196477462c725 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 12 Feb 2026 15:44:12 +0100 Subject: [PATCH 31/38] Add new validity convenience factories. --- libs/model/include/mapget/model/validity.h | 13 +++++++++ libs/model/src/validity.cpp | 32 ++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/libs/model/include/mapget/model/validity.h b/libs/model/include/mapget/model/validity.h index d941640b..f23bfb25 100644 --- a/libs/model/include/mapget/model/validity.h +++ b/libs/model/include/mapget/model/validity.h @@ -270,6 +270,19 @@ struct MultiValidity : public simfil::BaseArray model_ptr newGeometry(model_ptr, Validity::Direction direction = Validity::Empty); + /** + * Append a validity that references a feature ID without restricting the geometry. + * The referenced feature's geometry is resolved when the validity is evaluated. + */ + model_ptr + newFeatureId(model_ptr const& featureId, Validity::Direction direction = Validity::Empty); + + /** + * Append a validity that references a named geometry in the current feature context. + */ + model_ptr + newGeomName(std::string_view geomName, Validity::Direction direction = Validity::Empty); + /** * Append a direction validity without further restricting the range. * The direction value controls, in which direction along the referenced diff --git a/libs/model/src/validity.cpp b/libs/model/src/validity.cpp index 3f18445a..fa6229ab 100644 --- a/libs/model/src/validity.cpp +++ b/libs/model/src/validity.cpp @@ -279,6 +279,18 @@ SelfContainedGeometry Validity::computeGeometry( return true; }); + // If no geometry name is specified and no unnamed geometry was found, + // fall back to the first line geometry in the collection. + if (!geometry && !requiredGeomName) { + geometryCollection->forEachGeometry([&geometry](auto&& geom){ + if (geom->geomType() == GeomType::Line) { + geometry = geom; + return false; + } + return true; + }); + } + if (!geometry) { if (error) { *error = fmt::format("Failed to find geometry for {}", requiredGeomName ? *requiredGeomName : ""); @@ -459,6 +471,26 @@ MultiValidity::newGeometry(model_ptr geom, Validity::Direction directi return result; } +model_ptr +MultiValidity::newFeatureId(model_ptr const& featureId, Validity::Direction direction) +{ + auto result = model().newValidity(); + result->setFeatureId(featureId); + result->setDirection(direction); + append(result); + return result; +} + +model_ptr +MultiValidity::newGeomName(std::string_view geomName, Validity::Direction direction) +{ + auto result = model().newValidity(); + result->setGeometryName(geomName); + result->setDirection(direction); + append(result); + return result; +} + model_ptr MultiValidity::newDirection(Validity::Direction direction) { auto result = model().newValidity(); From c417117689b684b940b1d98cf089dfb7953f2335 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 12 Feb 2026 15:57:13 +0100 Subject: [PATCH 32/38] tiles-ws: add connection flow control and request context frames --- libs/http-service/src/tiles-ws-controller.cpp | 432 ++++++++++++++++-- libs/http-service/src/tiles-ws-controller.h | 4 +- libs/model/include/mapget/model/stream.h | 6 + 3 files changed, 400 insertions(+), 42 deletions(-) diff --git a/libs/http-service/src/tiles-ws-controller.cpp b/libs/http-service/src/tiles-ws-controller.cpp index 0651d3b1..5fa230a3 100644 --- a/libs/http-service/src/tiles-ws-controller.cpp +++ b/libs/http-service/src/tiles-ws-controller.cpp @@ -14,9 +14,11 @@ #include #include +#include #include #include #include +#include #include #include #include @@ -32,6 +34,40 @@ namespace mapget::detail namespace { +struct TilesWsMetrics +{ + std::atomic activeConnections{0}; + std::atomic activeSessions{0}; + std::atomic totalQueuedFrames{0}; + std::atomic totalQueuedBytes{0}; + std::atomic totalForwardedFrames{0}; + std::atomic totalForwardedBytes{0}; + std::atomic totalDroppedFrames{0}; + std::atomic totalDroppedBytes{0}; + std::atomic totalDrainCalls{0}; + std::atomic replacedRequests{0}; + std::atomic totalFlowGrantMessages{0}; + std::atomic totalFlowGrantFrames{0}; + std::atomic totalFlowGrantBytes{0}; + std::atomic totalFlowBlockedDrains{0}; +}; + +TilesWsMetrics gTilesWsMetrics; +std::mutex gTrackedSessionsMutex; +std::vector> gTrackedSessions; +std::mutex gTrackedConnectionsMutex; +std::vector> gTrackedConnections; + +constexpr std::string_view kFlowGrantType = "mapget.tiles.flow-grant"; +constexpr int64_t kFlowCreditMaxFrames = 16; +constexpr int64_t kFlowCreditMaxBytes = 64 * 1024 * 1024; + +[[nodiscard]] int64_t nonNegative(std::atomic const& value) +{ + const auto v = value.load(std::memory_order_relaxed); + return v < 0 ? 0 : v; +} + [[nodiscard]] AuthHeaders authHeadersFromRequest(const drogon::HttpRequestPtr& req) { AuthHeaders headers; @@ -84,11 +120,105 @@ namespace return message; } +[[nodiscard]] int64_t parseNonNegativeInt64(const nlohmann::json& j, std::string_view key) +{ + const auto keyString = std::string(key); + const auto it = j.find(keyString); + if (it == j.end()) { + return 0; + } + if (it->is_number_unsigned()) { + const auto raw = it->get(); + const auto max = static_cast(std::numeric_limits::max()); + return static_cast(std::min(raw, max)); + } + if (it->is_number_integer()) { + const auto raw = it->get(); + return std::max(0, raw); + } + return 0; +} + +[[nodiscard]] bool isFlowControlledDataFrameType(TileLayerStream::MessageType type) +{ + return type == TileLayerStream::MessageType::StringPool + || type == TileLayerStream::MessageType::TileFeatureLayer + || type == TileLayerStream::MessageType::TileSourceDataLayer; +} + +struct FlowControlStateSnapshot +{ + bool enabled = false; + int64_t creditFrames = 0; + int64_t creditBytes = 0; +}; + struct WsConnectionState { AuthHeaders authHeaders; TileLayerStream::StringPoolOffsetMap stringPoolOffsets; std::shared_ptr session; + uint64_t nextRequestId = 1; + + mutable std::mutex flowControlMutex; + bool flowControlEnabled = false; + int64_t flowCreditFrames = 0; + int64_t flowCreditBytes = 0; + + void setFlowControlEnabled(bool enabled) + { + std::lock_guard lock(flowControlMutex); + if (enabled) { + if (!flowControlEnabled) { + flowControlEnabled = true; + flowCreditFrames = kFlowCreditMaxFrames; + flowCreditBytes = kFlowCreditMaxBytes; + } + return; + } + flowControlEnabled = false; + flowCreditFrames = 0; + flowCreditBytes = 0; + } + + [[nodiscard]] std::pair grantFlowCredits(int64_t frames, int64_t bytes) + { + std::lock_guard lock(flowControlMutex); + if (!flowControlEnabled) { + return {0, 0}; + } + const auto safeFrames = std::max(0, frames); + const auto safeBytes = std::max(0, bytes); + const auto oldFrames = flowCreditFrames; + const auto oldBytes = flowCreditBytes; + flowCreditFrames = std::min(kFlowCreditMaxFrames, flowCreditFrames + safeFrames); + flowCreditBytes = std::min(kFlowCreditMaxBytes, flowCreditBytes + safeBytes); + return {flowCreditFrames - oldFrames, flowCreditBytes - oldBytes}; + } + + [[nodiscard]] bool consumeFlowCreditForFrame(int64_t frameSizeBytes) + { + std::lock_guard lock(flowControlMutex); + if (!flowControlEnabled) { + return true; + } + if (flowCreditFrames <= 0 || flowCreditBytes <= 0) { + return false; + } + flowCreditFrames -= 1; + flowCreditBytes = std::max(0, flowCreditBytes - std::max(0, frameSizeBytes)); + return true; + } + + [[nodiscard]] FlowControlStateSnapshot flowControlSnapshot() const + { + std::lock_guard lock(flowControlMutex); + return FlowControlStateSnapshot{ + .enabled = flowControlEnabled, + .creditFrames = flowCreditFrames, + .creditBytes = flowCreditBytes, + }; + } }; class TilesWsSession : public std::enable_shared_from_this @@ -98,12 +228,13 @@ class TilesWsSession : public std::enable_shared_from_this HttpService& service, std::weak_ptr conn, std::weak_ptr connState, + uint64_t requestId, AuthHeaders authHeaders, TileLayerStream::StringPoolOffsetMap initialOffsets) : service_(service), - loop_(drogon::app().getLoop()), conn_(std::move(conn)), connState_(std::move(connState)), + requestId_(requestId), authHeaders_(std::move(authHeaders)), offsets_(std::move(initialOffsets)), writer_( @@ -111,10 +242,12 @@ class TilesWsSession : public std::enable_shared_from_this [this](std::string msg, TileLayerStream::MessageType type) { onWriterMessage(std::move(msg), type); }, offsets_)) { + gTilesWsMetrics.activeSessions.fetch_add(1, std::memory_order_relaxed); } ~TilesWsSession() { + gTilesWsMetrics.activeSessions.fetch_sub(1, std::memory_order_relaxed); // Best-effort cleanup: abort any in-flight requests if the session is destroyed. cancelNoStatus(); } @@ -122,6 +255,28 @@ class TilesWsSession : public std::enable_shared_from_this TilesWsSession(TilesWsSession const&) = delete; TilesWsSession& operator=(TilesWsSession const&) = delete; + void registerForMetrics() + { + std::lock_guard lock(gTrackedSessionsMutex); + gTrackedSessions.push_back(weak_from_this()); + } + + [[nodiscard]] std::pair pendingSnapshot() + { + std::lock_guard lock(mutex_); + int64_t pendingFrames = static_cast(outgoing_.size()); + int64_t pendingBytes = 0; + for (auto const& frame : outgoing_) { + pendingBytes += static_cast(frame.bytes.size()); + } + return {pendingFrames, pendingBytes}; + } + + void onFlowGrant() + { + scheduleDrain(); + } + void start(const nlohmann::json& j) { auto requestsIt = j.find("requests"); @@ -189,6 +344,7 @@ class TilesWsSession : public std::enable_shared_from_this } // Start processing (may synchronously set request statuses). + queueRequestContextMessage(); (void)service_.request(requests_, authHeaders_); { @@ -206,7 +362,7 @@ class TilesWsSession : public std::enable_shared_from_this // Stop sending any queued tile frames from this session. { std::lock_guard lock(mutex_); - outgoing_.clear(); + clearOutgoingLocked(); } // Abort in-flight requests (best-effort). @@ -234,6 +390,7 @@ class TilesWsSession : public std::enable_shared_from_this struct OutgoingFrame { std::string bytes; + TileLayerStream::MessageType type{TileLayerStream::MessageType::None}; std::optional> stringPoolCommit; }; @@ -243,6 +400,32 @@ class TilesWsSession : public std::enable_shared_from_this TileLayerStream::MessageType type{TileLayerStream::MessageType::None}; }; + void enqueueOutgoingLocked(OutgoingFrame&& frame) + { + const auto bytes = static_cast(frame.bytes.size()); + outgoing_.push_back(std::move(frame)); + gTilesWsMetrics.totalQueuedFrames.fetch_add(1, std::memory_order_relaxed); + gTilesWsMetrics.totalQueuedBytes.fetch_add(bytes, std::memory_order_relaxed); + } + + void clearOutgoingLocked() + { + if (outgoing_.empty()) { + return; + } + + int64_t droppedFrames = 0; + int64_t droppedBytes = 0; + for (auto const& frame : outgoing_) { + ++droppedFrames; + droppedBytes += static_cast(frame.bytes.size()); + } + outgoing_.clear(); + + gTilesWsMetrics.totalDroppedFrames.fetch_add(droppedFrames, std::memory_order_relaxed); + gTilesWsMetrics.totalDroppedBytes.fetch_add(droppedBytes, std::memory_order_relaxed); + } + void cancelNoStatus() { if (cancelled_.exchange(true)) @@ -251,7 +434,7 @@ class TilesWsSession : public std::enable_shared_from_this // Ensure we stop emitting any further frames. { std::lock_guard lock(mutex_); - outgoing_.clear(); + clearOutgoingLocked(); } for (auto const& r : requests_) { @@ -309,10 +492,11 @@ class TilesWsSession : public std::enable_shared_from_this for (auto& m : batch) { OutgoingFrame frame; frame.bytes = std::move(m.bytes); + frame.type = m.type; if (m.type == TileLayerStream::MessageType::StringPool) { frame.stringPoolCommit = stringPoolCommit; } - outgoing_.push_back(std::move(frame)); + enqueueOutgoingLocked(std::move(frame)); } } @@ -347,9 +531,22 @@ class TilesWsSession : public std::enable_shared_from_this { OutgoingFrame frame; frame.bytes = encodeStreamMessage(TileLayerStream::MessageType::Status, buildStatusPayload(std::move(message))); + frame.type = TileLayerStream::MessageType::Status; + { + std::lock_guard lock(mutex_); + enqueueOutgoingLocked(std::move(frame)); + } + } + + void queueRequestContextMessage() + { + OutgoingFrame frame; + frame.bytes = + encodeStreamMessage(TileLayerStream::MessageType::RequestContext, buildRequestContextPayload()); + frame.type = TileLayerStream::MessageType::RequestContext; { std::lock_guard lock(mutex_); - outgoing_.push_back(std::move(frame)); + enqueueOutgoingLocked(std::move(frame)); } } @@ -362,9 +559,10 @@ class TilesWsSession : public std::enable_shared_from_this frame.bytes = encodeStreamMessage( TileLayerStream::MessageType::LoadStateChange, buildLoadStatePayload(key, state)); + frame.type = TileLayerStream::MessageType::LoadStateChange; { std::lock_guard lock(mutex_); - outgoing_.push_back(std::move(frame)); + enqueueOutgoingLocked(std::move(frame)); } scheduleDrain(); } @@ -397,16 +595,18 @@ class TilesWsSession : public std::enable_shared_from_this return nlohmann::json::object({ {"type", "mapget.tiles.status"}, + {"requestId", requestId_}, {"allDone", allDone}, {"requests", std::move(requestsJson)}, {"message", std::move(message)}, }).dump(); } - static std::string buildLoadStatePayload(MapTileKey const& key, TileLayer::LoadState state) + [[nodiscard]] std::string buildLoadStatePayload(MapTileKey const& key, TileLayer::LoadState state) const { return nlohmann::json::object({ {"type", "mapget.tiles.load-state"}, + {"requestId", requestId_}, {"mapId", key.mapId_}, {"layerId", key.layerId_}, {"tileId", key.tileId_.value_}, @@ -415,61 +615,103 @@ class TilesWsSession : public std::enable_shared_from_this }).dump(); } + [[nodiscard]] std::string buildRequestContextPayload() const + { + return nlohmann::json::object({ + {"type", "mapget.tiles.request-context"}, + {"requestId", requestId_}, + }).dump(); + } + void scheduleDrain() { if (drainScheduled_.exchange(true)) return; - - auto weak = weak_from_this(); - loop_->queueInLoop([weak = std::move(weak)]() mutable { - if (auto self = weak.lock()) { - self->drainOnLoop(); - } - }); + drainNow(); } - void drainOnLoop() + void drainNow() { - drainScheduled_ = false; + gTilesWsMetrics.totalDrainCalls.fetch_add(1, std::memory_order_relaxed); + + // Keep one active drainer at a time and bound each batch to avoid + // pushing very large bursts into Drogon's internal connection buffers. + for (;;) { + auto conn = conn_.lock(); + if (!conn || conn->disconnected()) { + drainScheduled_.store(false, std::memory_order_relaxed); + cancelNoStatus(); + return; + } - auto conn = conn_.lock(); - if (!conn || conn->disconnected()) { - cancelNoStatus(); - return; - } + constexpr size_t maxFramesPerDrain = 64; + constexpr size_t maxBytesPerDrain = 2 * 1024 * 1024; + size_t drainedBytes = 0; + bool blockedByFlowControl = false; - constexpr size_t maxFramesPerDrain = 256; - for (size_t i = 0; i < maxFramesPerDrain; ++i) { - OutgoingFrame frame; - { - std::lock_guard lock(mutex_); - if (outgoing_.empty()) { - break; + for (size_t i = 0; i < maxFramesPerDrain && drainedBytes < maxBytesPerDrain; ++i) { + OutgoingFrame frame; + { + std::lock_guard lock(mutex_); + if (outgoing_.empty()) { + break; + } + frame = std::move(outgoing_.front()); + outgoing_.pop_front(); + } + + const auto frameBytes = static_cast(frame.bytes.size()); + + if (cancelled_) { + gTilesWsMetrics.totalDroppedFrames.fetch_add(1, std::memory_order_relaxed); + gTilesWsMetrics.totalDroppedBytes.fetch_add(frameBytes, std::memory_order_relaxed); + continue; + } + + if (isFlowControlledDataFrameType(frame.type)) { + auto state = connState_.lock(); + if (!state || !state->consumeFlowCreditForFrame(frameBytes)) { + std::lock_guard lock(mutex_); + outgoing_.push_front(std::move(frame)); + blockedByFlowControl = true; + break; + } } - frame = std::move(outgoing_.front()); - outgoing_.pop_front(); - } - conn->send(frame.bytes, drogon::WebSocketMessageType::Binary); - if (frame.stringPoolCommit) { - if (auto state = connState_.lock()) { - state->stringPoolOffsets[frame.stringPoolCommit->first] = frame.stringPoolCommit->second; + drainedBytes += static_cast(frameBytes); + gTilesWsMetrics.totalForwardedFrames.fetch_add(1, std::memory_order_relaxed); + gTilesWsMetrics.totalForwardedBytes.fetch_add(frameBytes, std::memory_order_relaxed); + conn->send(frame.bytes, drogon::WebSocketMessageType::Binary); + if (frame.stringPoolCommit) { + if (auto state = connState_.lock()) { + state->stringPoolOffsets[frame.stringPoolCommit->first] = frame.stringPoolCommit->second; + } } } - } - { - std::lock_guard lock(mutex_); - if (outgoing_.empty()) + bool done = false; + { + std::lock_guard lock(mutex_); + if (blockedByFlowControl || outgoing_.empty()) { + // Release ownership only while holding mutex_ so enqueuers can + // reliably schedule a new drain for subsequently queued frames. + drainScheduled_.store(false, std::memory_order_relaxed); + done = true; + } + } + if (blockedByFlowControl) { + gTilesWsMetrics.totalFlowBlockedDrains.fetch_add(1, std::memory_order_relaxed); + } + if (done) { return; + } } - scheduleDrain(); } HttpService& service_; - trantor::EventLoop* loop_; std::weak_ptr conn_; std::weak_ptr connState_; + uint64_t requestId_; AuthHeaders authHeaders_; @@ -495,8 +737,13 @@ class TilesWebSocketController final : public drogon::WebSocketController(); state->authHeaders = authHeadersFromRequest(req); + { + std::lock_guard lock(gTrackedConnectionsMutex); + gTrackedConnections.push_back(state); + } conn->setContext(std::move(state)); } @@ -537,6 +784,30 @@ class TilesWebSocketController final : public drogon::WebSocketControlleris_string()) { + messageType = typeIt->get(); + } + + if (messageType == kFlowGrantType) { + auto [grantedFrames, grantedBytes] = state->grantFlowCredits( + parseNonNegativeInt64(j, "frames"), + parseNonNegativeInt64(j, "bytes")); + gTilesWsMetrics.totalFlowGrantMessages.fetch_add(1, std::memory_order_relaxed); + gTilesWsMetrics.totalFlowGrantFrames.fetch_add(grantedFrames, std::memory_order_relaxed); + gTilesWsMetrics.totalFlowGrantBytes.fetch_add(grantedBytes, std::memory_order_relaxed); + if (state->session) { + state->session->onFlowGrant(); + } + return; + } + + bool flowControl = false; + if (auto flowControlIt = j.find("flowControl"); flowControlIt != j.end() && flowControlIt->is_boolean()) { + flowControl = flowControlIt->get(); + } + state->setFlowControlEnabled(flowControl); + // Patch per-connection string pool offsets if supplied. if (j.contains("stringPoolOffsets")) { if (!j["stringPoolOffsets"].is_object()) { @@ -567,21 +838,35 @@ class TilesWebSocketController final : public drogon::WebSocketControllersession) { + gTilesWsMetrics.replacedRequests.fetch_add(1, std::memory_order_relaxed); state->session->cancel("Replaced by a new /tiles WebSocket request."); state->session.reset(); } + uint64_t requestId = state->nextRequestId++; + if (auto requestIdIt = j.find("requestId"); + requestIdIt != j.end() && (requestIdIt->is_number_integer() || requestIdIt->is_number_unsigned())) { + const auto parsedRequestId = parseNonNegativeInt64(j, "requestId"); + if (parsedRequestId > 0) { + requestId = static_cast(parsedRequestId); + state->nextRequestId = std::max(state->nextRequestId, requestId + 1); + } + } + state->session = std::make_shared( service_, conn, state, + requestId, state->authHeaders, state->stringPoolOffsets); + state->session->registerForMetrics(); state->session->start(j); } void handleConnectionClosed(const drogon::WebSocketConnectionPtr& conn) override { + gTilesWsMetrics.activeConnections.fetch_sub(1, std::memory_order_relaxed); if (auto state = conn->getContext()) { if (state->session) { state->session->cancel("WebSocket connection closed."); @@ -604,4 +889,69 @@ void registerTilesWebSocketController(drogon::HttpAppFramework& app, HttpService app.registerController(std::make_shared(service)); } +nlohmann::json tilesWebSocketMetricsSnapshot() +{ + int64_t pendingControllerFrames = 0; + int64_t pendingControllerBytes = 0; + int64_t flowControlEnabledConnections = 0; + int64_t flowControlBlockedConnections = 0; + int64_t flowControlCreditFrames = 0; + int64_t flowControlCreditBytes = 0; + { + std::lock_guard lock(gTrackedSessionsMutex); + auto out = gTrackedSessions.begin(); + for (auto it = gTrackedSessions.begin(); it != gTrackedSessions.end(); ++it) { + if (auto session = it->lock()) { + auto [frames, bytes] = session->pendingSnapshot(); + pendingControllerFrames += frames; + pendingControllerBytes += bytes; + *out++ = *it; + } + } + gTrackedSessions.erase(out, gTrackedSessions.end()); + } + { + std::lock_guard lock(gTrackedConnectionsMutex); + auto out = gTrackedConnections.begin(); + for (auto it = gTrackedConnections.begin(); it != gTrackedConnections.end(); ++it) { + if (auto state = it->lock()) { + const auto snapshot = state->flowControlSnapshot(); + if (snapshot.enabled) { + ++flowControlEnabledConnections; + flowControlCreditFrames += snapshot.creditFrames; + flowControlCreditBytes += snapshot.creditBytes; + if (snapshot.creditFrames <= 0 || snapshot.creditBytes <= 0) { + ++flowControlBlockedConnections; + } + } + *out++ = *it; + } + } + gTrackedConnections.erase(out, gTrackedConnections.end()); + } + + return nlohmann::json::object({ + {"active-connections", nonNegative(gTilesWsMetrics.activeConnections)}, + {"active-sessions", nonNegative(gTilesWsMetrics.activeSessions)}, + {"pending-controller-frames", pendingControllerFrames}, + {"pending-controller-bytes", pendingControllerBytes}, + {"flow-control-enabled-connections", flowControlEnabledConnections}, + {"flow-control-blocked-connections", flowControlBlockedConnections}, + {"flow-control-credit-frames", flowControlCreditFrames}, + {"flow-control-credit-bytes", flowControlCreditBytes}, + {"total-queued-frames", nonNegative(gTilesWsMetrics.totalQueuedFrames)}, + {"total-queued-bytes", nonNegative(gTilesWsMetrics.totalQueuedBytes)}, + {"total-forwarded-frames", nonNegative(gTilesWsMetrics.totalForwardedFrames)}, + {"total-forwarded-bytes", nonNegative(gTilesWsMetrics.totalForwardedBytes)}, + {"total-dropped-frames", nonNegative(gTilesWsMetrics.totalDroppedFrames)}, + {"total-dropped-bytes", nonNegative(gTilesWsMetrics.totalDroppedBytes)}, + {"total-drain-calls", nonNegative(gTilesWsMetrics.totalDrainCalls)}, + {"total-flow-grant-messages", nonNegative(gTilesWsMetrics.totalFlowGrantMessages)}, + {"total-flow-grant-frames", nonNegative(gTilesWsMetrics.totalFlowGrantFrames)}, + {"total-flow-grant-bytes", nonNegative(gTilesWsMetrics.totalFlowGrantBytes)}, + {"total-flow-blocked-drains", nonNegative(gTilesWsMetrics.totalFlowBlockedDrains)}, + {"replaced-requests", nonNegative(gTilesWsMetrics.replacedRequests)}, + }); +} + } // namespace mapget::detail diff --git a/libs/http-service/src/tiles-ws-controller.h b/libs/http-service/src/tiles-ws-controller.h index 4acaed45..7f274e6a 100644 --- a/libs/http-service/src/tiles-ws-controller.h +++ b/libs/http-service/src/tiles-ws-controller.h @@ -1,5 +1,7 @@ #pragma once +#include "nlohmann/json_fwd.hpp" + namespace drogon { class HttpAppFramework; @@ -14,6 +16,6 @@ namespace mapget::detail { void registerTilesWebSocketController(drogon::HttpAppFramework& app, HttpService& service); +[[nodiscard]] nlohmann::json tilesWebSocketMetricsSnapshot(); } // namespace mapget::detail - diff --git a/libs/model/include/mapget/model/stream.h b/libs/model/include/mapget/model/stream.h index 5eb0a5a5..a219564f 100644 --- a/libs/model/include/mapget/model/stream.h +++ b/libs/model/include/mapget/model/stream.h @@ -41,6 +41,12 @@ class TileLayerStream * Payload: UTF-8 JSON bytes (not null-terminated). */ LoadStateChange = 5, + /** + * JSON-encoded request-context marker for WebSocket /tiles streams. + * + * Payload: UTF-8 JSON bytes (not null-terminated). + */ + RequestContext = 6, EndOfStream = 128 }; From 31bce7773124de7e6e58f21d03bf67eab79de1d7 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 12 Feb 2026 15:57:16 +0100 Subject: [PATCH 33/38] status: add live dashboard and optional heavy stats endpoint --- libs/http-service/src/http-service-impl.h | 4 + libs/http-service/src/http-service.cpp | 7 + libs/http-service/src/status-handler.cpp | 470 +++++++++++++++++- libs/service/include/mapget/service/service.h | 11 + libs/service/src/service.cpp | 200 ++++++-- 5 files changed, 627 insertions(+), 65 deletions(-) diff --git a/libs/http-service/src/http-service-impl.h b/libs/http-service/src/http-service-impl.h index 2db77a6c..88d2d1af 100644 --- a/libs/http-service/src/http-service-impl.h +++ b/libs/http-service/src/http-service-impl.h @@ -54,6 +54,10 @@ struct HttpService::Impl const drogon::HttpRequestPtr& req, std::function&& callback) const; + void handleStatusDataRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const; + void handleLocateRequest( const drogon::HttpRequestPtr& req, std::function&& callback) const; diff --git a/libs/http-service/src/http-service.cpp b/libs/http-service/src/http-service.cpp index 4e377448..bc46f860 100644 --- a/libs/http-service/src/http-service.cpp +++ b/libs/http-service/src/http-service.cpp @@ -43,6 +43,13 @@ void HttpService::setup(drogon::HttpAppFramework& app) }, {drogon::Get}); + app.registerHandler( + "/status-data", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) { + impl_->handleStatusDataRequest(req, std::move(callback)); + }, + {drogon::Get}); + app.registerHandler( "/locate", [this](const drogon::HttpRequestPtr& req, std::function&& callback) { diff --git a/libs/http-service/src/status-handler.cpp b/libs/http-service/src/status-handler.cpp index b0a91216..2992b21e 100644 --- a/libs/http-service/src/status-handler.cpp +++ b/libs/http-service/src/status-handler.cpp @@ -1,34 +1,474 @@ #include "http-service-impl.h" +#include "tiles-ws-controller.h" + #include -#include +#include +#include +#include namespace mapget { +namespace +{ + +[[nodiscard]] bool parseBoolParameter(const drogon::HttpRequestPtr& req, std::string_view key, bool defaultValue = false) +{ + const std::string raw = req->getParameter(std::string(key)); + if (raw.empty()) + return defaultValue; + + if (raw == "1" || raw == "true" || raw == "TRUE" || raw == "yes" || raw == "on") { + return true; + } + if (raw == "0" || raw == "false" || raw == "FALSE" || raw == "no" || raw == "off") { + return false; + } + return defaultValue; +} + +[[nodiscard]] std::string statusPageHtml() +{ + return R"HTML( + + + + + +mapget status + + + +

Status Information

+
+ + + + Never updated + +
+ +

Tiles WebSocket Metrics

+
+ + + +
MetricValue
+
+ `pending-controller-*` covers frames still queued in mapget's tiles websocket controller. + `total-forwarded-*` counts frames already handed to Drogon via `conn->send(...)`. + `flow-control-credit-*` shows currently available connection-level send credits. + `flow-control-blocked-connections` counts flow-controlled connections currently blocked by zero frame or byte credits. +
+
+ + + +

Service Statistics

+
+ +

Cache Statistics

+
+ +
+

Tile Size Distribution

+
+ + + +
MetricValue
+
+
+ + + +
BinCountShare
+
+
+ + + + +)HTML"; +} + +} // namespace + +void HttpService::Impl::handleStatusDataRequest( + const drogon::HttpRequestPtr& req, + std::function&& callback) const +{ + const bool includeTileSizeDistribution = + parseBoolParameter(req, "includeTileSizeDistribution", false); + const bool includeCachedFeatureTreeBytes = + parseBoolParameter(req, "includeCachedFeatureTreeBytes", true); + + const auto payload = nlohmann::json::object({ + {"timestampMs", + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()}, + {"service", self_.getStatistics(includeCachedFeatureTreeBytes, includeTileSizeDistribution)}, + {"cache", self_.cache()->getStatistics()}, + {"tilesWebsocket", detail::tilesWebSocketMetricsSnapshot()}, + }); + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(payload.dump()); + callback(resp); +} void HttpService::Impl::handleStatusRequest( const drogon::HttpRequestPtr& /*req*/, std::function&& callback) const { - auto serviceStats = self_.getStatistics(); - auto cacheStats = self_.cache()->getStatistics(); - - std::ostringstream oss; - oss << ""; - oss << "

Status Information

"; - oss << "

Service Statistics

"; - oss << "
" << serviceStats.dump(4) << "
"; - oss << "

Cache Statistics

"; - oss << "
" << cacheStats.dump(4) << "
"; - oss << ""; - auto resp = drogon::HttpResponse::newHttpResponse(); resp->setStatusCode(drogon::k200OK); resp->setContentTypeCode(drogon::CT_TEXT_HTML); - resp->setBody(oss.str()); + resp->setBody(statusPageHtml()); callback(resp); } } // namespace mapget - diff --git a/libs/service/include/mapget/service/service.h b/libs/service/include/mapget/service/service.h index cf1cb085..6ad41df9 100644 --- a/libs/service/include/mapget/service/service.h +++ b/libs/service/include/mapget/service/service.h @@ -206,6 +206,17 @@ class Service */ [[nodiscard]] nlohmann::json getStatistics() const; + /** + * Variant of getStatistics() with optional expensive analyses: + * - includeCachedFeatureTreeBytes: Parse cached feature tiles and aggregate + * detailed subtree sizes. + * - includeTileSizeDistribution: Build cached feature-tile size histogram + * and percentiles. + */ + [[nodiscard]] nlohmann::json getStatistics( + bool includeCachedFeatureTreeBytes, + bool includeTileSizeDistribution) const; + /** Get the Cache which this service was constructed with. */ [[nodiscard]] Cache::Ptr cache(); diff --git a/libs/service/src/service.cpp b/libs/service/src/service.cpp index 7d58d206..0069b178 100644 --- a/libs/service/src/service.cpp +++ b/libs/service/src/service.cpp @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include "simfil/types.h" @@ -743,7 +745,83 @@ RequestStatus Service::hasLayerAndCanAccess( return RequestStatus::NoDataSource; } +namespace +{ + +[[nodiscard]] nlohmann::json buildTileSizeDistribution(std::vector tileSizes) +{ + if (tileSizes.empty()) + return nlohmann::json::object(); + + std::sort(tileSizes.begin(), tileSizes.end()); + + const int64_t totalBytes = std::accumulate(tileSizes.begin(), tileSizes.end(), int64_t{0}); + const int64_t tileCount = static_cast(tileSizes.size()); + const int64_t meanBytes = totalBytes / tileCount; + + struct HistogramBin { + int64_t upperBound; + const char* label; + int64_t count = 0; + }; + std::vector bins = { + {16 * 1024, "<=16 KiB"}, + {32 * 1024, "16-32 KiB"}, + {64 * 1024, "32-64 KiB"}, + {128 * 1024, "64-128 KiB"}, + {256 * 1024, "128-256 KiB"}, + {512 * 1024, "256-512 KiB"}, + {1024 * 1024, "512 KiB-1 MiB"}, + {2 * 1024 * 1024, "1-2 MiB"}, + {4 * 1024 * 1024, "2-4 MiB"}, + }; + int64_t overflowCount = 0; + + for (const auto bytes : tileSizes) { + bool assigned = false; + for (auto& bin : bins) { + if (bytes <= bin.upperBound) { + ++bin.count; + assigned = true; + break; + } + } + if (!assigned) { + ++overflowCount; + } + } + + auto histogram = nlohmann::json::array(); + for (const auto& bin : bins) { + histogram.push_back(nlohmann::json::object({ + {"label", bin.label}, + {"count", bin.count}, + })); + } + histogram.push_back(nlohmann::json::object({ + {"label", ">4 MiB"}, + {"count", overflowCount}, + })); + + return nlohmann::json::object({ + {"tile-count", tileCount}, + {"total-tile-bytes", totalBytes}, + {"min-bytes", tileSizes.front()}, + {"max-bytes", tileSizes.back()}, + {"mean-bytes", meanBytes}, + {"histogram", std::move(histogram)}, + }); +} + +} // namespace + nlohmann::json Service::getStatistics() const +{ + // Preserve old behavior for existing callers. + return getStatistics(true, false); +} + +nlohmann::json Service::getStatistics(bool includeCachedFeatureTreeBytes, bool includeTileSizeDistribution) const { auto datasources = nlohmann::json::array(); for (auto const& [dataSource, info] : impl_->dataSourceInfo_) { @@ -758,73 +836,91 @@ nlohmann::json Service::getStatistics() const {"active-requests", impl_->requests_.size()} }; - auto layerInfoByMap = std::unordered_map>>{}; - for (auto const& [_, info] : impl_->dataSourceInfo_) { - auto& layers = layerInfoByMap[info.mapId_]; - for (auto const& [layerId, layerInfo] : info.layers_) { - layers[layerId] = layerInfo; - } + if (!includeCachedFeatureTreeBytes && !includeTileSizeDistribution) { + return result; } - auto resolveLayerInfo = [&](std::string_view mapId, std::string_view layerId) -> std::shared_ptr { - auto mapIt = layerInfoByMap.find(std::string(mapId)); - if (mapIt == layerInfoByMap.end()) - return std::make_shared(); - auto layerIt = mapIt->second.find(std::string(layerId)); - if (layerIt == mapIt->second.end()) { - auto fallback = std::make_shared(); - fallback->layerId_ = std::string(layerId); - return fallback; - } - return layerIt->second; - }; - + auto featureLayerTotals = nlohmann::json::object(); + auto modelPoolTotals = nlohmann::json::object(); int64_t parsedTiles = 0; int64_t totalTileBytes = 0; int64_t parseErrors = 0; - auto featureLayerTotals = nlohmann::json::object(); - auto modelPoolTotals = nlohmann::json::object(); + std::vector tileSizes; auto addTotals = [](nlohmann::json& totals, const nlohmann::json& stats) { for (const auto& [key, value] : stats.items()) { - if (value.is_number_integer()) - { + if (value.is_number_integer()) { totals[key] = totals.value(key, 0) + value.get(); - } - else if (value.is_number_float()) - { + } else if (value.is_number_float()) { totals[key] = totals.value(key, .0) + value.get(); } } }; - TileLayerStream::Reader tileReader( - resolveLayerInfo, - [&](auto&& parsedLayer) - { - auto tile = std::static_pointer_cast(parsedLayer); - auto sizeStats = tile->serializationSizeStats(); - addTotals(featureLayerTotals, sizeStats["feature-layer"]); - addTotals(modelPoolTotals, sizeStats["model-pool"]); - }, - impl_->cache_); - - impl_->cache_->forEachTileLayerBlob( - [&](const MapTileKey& key, const std::string& blob) - { - if (key.layer_ != LayerType::Features) - return; - ++parsedTiles; - totalTileBytes += static_cast(blob.size()); - try { - tileReader.read(blob); + std::unique_ptr tileReader; + if (includeCachedFeatureTreeBytes) { + auto layerInfoByMap = + std::unordered_map>>{}; + for (auto const& [_, info] : impl_->dataSourceInfo_) { + auto& layers = layerInfoByMap[info.mapId_]; + for (auto const& [layerId, layerInfo] : info.layers_) { + layers[layerId] = layerInfo; } - catch (const std::exception&) { - ++parseErrors; + } + + auto resolveLayerInfo = [layerInfoByMap](std::string_view mapId, std::string_view layerId) + -> std::shared_ptr { + auto mapIt = layerInfoByMap.find(std::string(mapId)); + if (mapIt == layerInfoByMap.end()) + return std::make_shared(); + auto layerIt = mapIt->second.find(std::string(layerId)); + if (layerIt == mapIt->second.end()) { + auto fallback = std::make_shared(); + fallback->layerId_ = std::string(layerId); + return fallback; } - }); + return layerIt->second; + }; - if (parsedTiles > 0) { + tileReader = std::make_unique( + resolveLayerInfo, + [&](auto&& parsedLayer) { + auto tile = std::dynamic_pointer_cast(parsedLayer); + if (!tile) { + ++parseErrors; + return; + } + auto sizeStats = tile->serializationSizeStats(); + addTotals(featureLayerTotals, sizeStats["feature-layer"]); + addTotals(modelPoolTotals, sizeStats["model-pool"]); + }, + impl_->cache_); + } + + impl_->cache_->forEachTileLayerBlob([&](const MapTileKey& key, const std::string& blob) { + if (key.layer_ != LayerType::Features) + return; + + const int64_t tileBytes = static_cast(blob.size()); + ++parsedTiles; + totalTileBytes += tileBytes; + + if (includeTileSizeDistribution) { + tileSizes.push_back(tileBytes); + } + + if (!includeCachedFeatureTreeBytes) { + return; + } + + try { + tileReader->read(blob); + } catch (const std::exception&) { + ++parseErrors; + } + }); + + if (includeCachedFeatureTreeBytes && parsedTiles > 0) { result["cached-feature-tree-bytes"] = nlohmann::json{ {"tile-count", parsedTiles}, {"total-tile-bytes", totalTileBytes}, @@ -834,6 +930,10 @@ nlohmann::json Service::getStatistics() const }; } + if (includeTileSizeDistribution && !tileSizes.empty()) { + result["cached-feature-tile-size-distribution"] = buildTileSizeDistribution(std::move(tileSizes)); + } + return result; } From 745943caefcc3bdf998efe0c40c59bb8ac661e47 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 12 Feb 2026 15:57:20 +0100 Subject: [PATCH 34/38] model: fix MapTileKey layer parsing index --- libs/model/src/layer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/model/src/layer.cpp b/libs/model/src/layer.cpp index 199851c7..a664acae 100644 --- a/libs/model/src/layer.cpp +++ b/libs/model/src/layer.cpp @@ -28,7 +28,7 @@ MapTileKey::MapTileKey(const std::string& str) if (partsVec.size() < 4) raise(fmt::format("Invalid cache tile id: {}", str)); - layer_ = nlohmann::json(std::string_view(&*partsVec[1].begin(), distance(partsVec[1]))).get(); + layer_ = nlohmann::json(std::string_view(&*partsVec[0].begin(), distance(partsVec[0]))).get(); mapId_ = std::string_view(&*partsVec[1].begin(), distance(partsVec[1])); layerId_ = std::string_view(&*partsVec[2].begin(), distance(partsVec[2])); std::from_chars(&*partsVec[3].begin(), &*partsVec[3].begin() + distance(partsVec[3]), tileId_.value_, 16); From 8e7de2b4a0abfc047f18cb2733553936d9aac519 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Fri, 13 Feb 2026 08:49:23 +0100 Subject: [PATCH 35/38] Lower in-fligth frame allowance to two for increased responsiveness. --- libs/http-service/src/tiles-ws-controller.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/http-service/src/tiles-ws-controller.cpp b/libs/http-service/src/tiles-ws-controller.cpp index 5fa230a3..56a69079 100644 --- a/libs/http-service/src/tiles-ws-controller.cpp +++ b/libs/http-service/src/tiles-ws-controller.cpp @@ -59,7 +59,7 @@ std::mutex gTrackedConnectionsMutex; std::vector> gTrackedConnections; constexpr std::string_view kFlowGrantType = "mapget.tiles.flow-grant"; -constexpr int64_t kFlowCreditMaxFrames = 16; +constexpr int64_t kFlowCreditMaxFrames = 2; constexpr int64_t kFlowCreditMaxBytes = 64 * 1024 * 1024; [[nodiscard]] int64_t nonNegative(std::atomic const& value) From d5394bb70264b676f383a6e3434b31f6049a8f76 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 16 Feb 2026 12:53:15 +0100 Subject: [PATCH 36/38] Use simfil release branch. --- cmake/deps.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/deps.cmake b/cmake/deps.cmake index 89300df1..20ba0220 100644 --- a/cmake/deps.cmake +++ b/cmake/deps.cmake @@ -15,7 +15,7 @@ CPMAddPackage( "EXPECTED_BUILD_TESTS OFF" "EXPECTED_BUILD_PACKAGE_DEB OFF") CPMAddPackage( - URI "gh:Klebert-Engineering/simfil#byte-array" + URI "gh:Klebert-Engineering/simfil#v0.6.3" OPTIONS "SIMFIL_WITH_MODEL_JSON ON" "SIMFIL_SHARED OFF") From b8f33d51113c040d1b4ff38397e43000e56db48c Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Tue, 17 Feb 2026 07:29:57 +0100 Subject: [PATCH 37/38] status: fix refresh race and frame-only ws metrics --- libs/http-service/src/status-handler.cpp | 26 ++++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/libs/http-service/src/status-handler.cpp b/libs/http-service/src/status-handler.cpp index 2992b21e..572c3b01 100644 --- a/libs/http-service/src/status-handler.cpp +++ b/libs/http-service/src/status-handler.cpp @@ -108,8 +108,8 @@ th { background: #f1f5f9; }
`pending-controller-*` covers frames still queued in mapget's tiles websocket controller. `total-forwarded-*` counts frames already handed to Drogon via `conn->send(...)`. - `flow-control-credit-*` shows currently available connection-level send credits. - `flow-control-blocked-connections` counts flow-controlled connections currently blocked by zero frame or byte credits. + `flow-control-credit-frames` shows currently available connection-level frame credits. + `flow-control-blocked-connections` counts flow-controlled connections currently blocked by zero frame credits.
@@ -179,6 +179,7 @@ const formatBytes = (bytes) => { const state = { timer: null, refreshInFlight: false, + pendingForcedRefresh: false, lastServiceText: "", lastCacheText: "", lastBreakdownJson: "", @@ -193,7 +194,6 @@ const wsMetricDefinitions = [ ["flow-control-enabled-connections", "flow-control-enabled-connections", (v) => formatInt(v)], ["flow-control-blocked-connections", "flow-control-blocked-connections", (v) => formatInt(v)], ["flow-control-credit-frames", "flow-control-credit-frames", (v) => formatInt(v)], - ["flow-control-credit-bytes", "flow-control-credit-bytes", (v) => `${formatInt(v)} (${formatBytes(v)})`], ["total-queued-frames", "total-queued-frames", (v) => formatInt(v)], ["total-queued-bytes", "total-queued-bytes", (v) => `${formatInt(v)} (${formatBytes(v)})`], ["total-forwarded-frames", "total-forwarded-frames", (v) => formatInt(v)], @@ -203,7 +203,6 @@ const wsMetricDefinitions = [ ["total-drain-calls", "total-drain-calls", (v) => formatInt(v)], ["total-flow-grant-messages", "total-flow-grant-messages", (v) => formatInt(v)], ["total-flow-grant-frames", "total-flow-grant-frames", (v) => formatInt(v)], - ["total-flow-grant-bytes", "total-flow-grant-bytes", (v) => `${formatInt(v)} (${formatBytes(v)})`], ["total-flow-blocked-drains", "total-flow-blocked-drains", (v) => formatInt(v)], ["replaced-requests", "replaced-requests", (v) => formatInt(v)], ]; @@ -371,8 +370,11 @@ function renderTileDistribution(service) { } } -async function refreshStatus() { +async function refreshStatus(force = false) { if (state.refreshInFlight) { + if (force) { + state.pendingForcedRefresh = true; + } return; } state.refreshInFlight = true; @@ -382,10 +384,12 @@ async function refreshStatus() { } try { const includeTileSizeDistribution = !!byId("includeTileSizeDistribution")?.checked; + const includeCachedFeatureTreeBytes = includeTileSizeDistribution; const params = new URLSearchParams(); if (includeTileSizeDistribution) { params.set("includeTileSizeDistribution", "1"); } + params.set("includeCachedFeatureTreeBytes", includeCachedFeatureTreeBytes ? "1" : "0"); params.set("_", String(Date.now())); const response = await fetch(`/status-data?${params.toString()}`, {cache: "no-store"}); @@ -409,6 +413,10 @@ async function refreshStatus() { } } finally { state.refreshInFlight = false; + if (state.pendingForcedRefresh) { + state.pendingForcedRefresh = false; + queueMicrotask(() => refreshStatus(false)); + } } } @@ -418,14 +426,14 @@ function resetTimer() { } const refreshMsInput = byId("refreshMs"); const interval = Math.max(200, Number(refreshMsInput?.value || 1000)); - state.timer = setInterval(refreshStatus, interval); + state.timer = setInterval(() => refreshStatus(false), interval); } byId("refreshMs")?.addEventListener("change", resetTimer); -byId("refreshNow")?.addEventListener("click", refreshStatus); -byId("includeTileSizeDistribution")?.addEventListener("change", refreshStatus); +byId("refreshNow")?.addEventListener("click", () => refreshStatus(true)); +byId("includeTileSizeDistribution")?.addEventListener("change", () => refreshStatus(true)); resetTimer(); -refreshStatus(); +refreshStatus(true); From 300c5aecf0ed961b3b523a7c19a5abd827094b58 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Tue, 17 Feb 2026 07:30:04 +0100 Subject: [PATCH 38/38] tiles-ws: add frame-credit flow control and request-aware queueing --- libs/http-service/src/tiles-ws-controller.cpp | 790 ++++++++++++------ 1 file changed, 555 insertions(+), 235 deletions(-) diff --git a/libs/http-service/src/tiles-ws-controller.cpp b/libs/http-service/src/tiles-ws-controller.cpp index 56a69079..21474420 100644 --- a/libs/http-service/src/tiles-ws-controller.cpp +++ b/libs/http-service/src/tiles-ws-controller.cpp @@ -19,8 +19,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -48,26 +50,28 @@ struct TilesWsMetrics std::atomic replacedRequests{0}; std::atomic totalFlowGrantMessages{0}; std::atomic totalFlowGrantFrames{0}; - std::atomic totalFlowGrantBytes{0}; std::atomic totalFlowBlockedDrains{0}; }; TilesWsMetrics gTilesWsMetrics; std::mutex gTrackedSessionsMutex; std::vector> gTrackedSessions; -std::mutex gTrackedConnectionsMutex; -std::vector> gTrackedConnections; -constexpr std::string_view kFlowGrantType = "mapget.tiles.flow-grant"; -constexpr int64_t kFlowCreditMaxFrames = 2; -constexpr int64_t kFlowCreditMaxBytes = 64 * 1024 * 1024; +constexpr std::string_view FLOW_GRANT_TYPE = "mapget.tiles.flow-grant"; +constexpr int64_t FLOW_CREDIT_MAX_FRAMES = 2; +constexpr size_t MAX_FRAMES_PER_DRAIN = 64; +constexpr LayerType REQUEST_TILE_LAYER_TYPE = LayerType::Features; +constexpr int64_t LOWEST_TILE_PRIORITY = std::numeric_limits::max(); +constexpr bool EMIT_LOAD_STATE_FRAMES = false; +/// Clamp an atomic metric value to zero to avoid exposing negative snapshots. [[nodiscard]] int64_t nonNegative(std::atomic const& value) { const auto v = value.load(std::memory_order_relaxed); return v < 0 ? 0 : v; } +/// Copy inbound HTTP headers so backend requests can preserve auth context. [[nodiscard]] AuthHeaders authHeadersFromRequest(const drogon::HttpRequestPtr& req) { AuthHeaders headers; @@ -77,6 +81,7 @@ constexpr int64_t kFlowCreditMaxBytes = 64 * 1024 * 1024; return headers; } +/// Convert internal request status enum values to stable UI-facing strings. [[nodiscard]] std::string_view requestStatusToString(RequestStatus s) { switch (s) { @@ -94,6 +99,7 @@ constexpr int64_t kFlowCreditMaxBytes = 64 * 1024 * 1024; return "Unknown"; } +/// Convert tile load-state enum values to stable UI-facing strings. [[nodiscard]] std::string_view loadStateToString(TileLayer::LoadState s) { switch (s) { @@ -107,6 +113,7 @@ constexpr int64_t kFlowCreditMaxBytes = 64 * 1024 * 1024; return "Unknown"; } +/// Encode one mapget VTLV frame with protocol header plus payload bytes. [[nodiscard]] std::string encodeStreamMessage(TileLayerStream::MessageType type, std::string_view payload) { std::ostringstream headerStream; @@ -120,6 +127,7 @@ constexpr int64_t kFlowCreditMaxBytes = 64 * 1024 * 1024; return message; } +/// Parse a JSON numeric field into non-negative int64 while handling missing keys. [[nodiscard]] int64_t parseNonNegativeInt64(const nlohmann::json& j, std::string_view key) { const auto keyString = std::string(key); @@ -139,6 +147,7 @@ constexpr int64_t kFlowCreditMaxBytes = 64 * 1024 * 1024; return 0; } +/// Return true for frame kinds governed by websocket flow-control credits. [[nodiscard]] bool isFlowControlledDataFrameType(TileLayerStream::MessageType type) { return type == TileLayerStream::MessageType::StringPool @@ -146,105 +155,53 @@ constexpr int64_t kFlowCreditMaxBytes = 64 * 1024 * 1024; || type == TileLayerStream::MessageType::TileSourceDataLayer; } -struct FlowControlStateSnapshot +/// Build a canonical request key using map/layer/tile while normalizing layer type. +[[nodiscard]] MapTileKey makeCanonicalRequestedTileKey( + std::string_view mapId, + std::string_view layerId, + TileId tileId) { - bool enabled = false; - int64_t creditFrames = 0; - int64_t creditBytes = 0; -}; + return MapTileKey( + REQUEST_TILE_LAYER_TYPE, + std::string(mapId), + std::string(layerId), + tileId); +} -struct WsConnectionState +/// Normalize an existing map tile key so request matching ignores source layer type. +[[nodiscard]] MapTileKey makeCanonicalRequestedTileKey(MapTileKey key) { - AuthHeaders authHeaders; - TileLayerStream::StringPoolOffsetMap stringPoolOffsets; - std::shared_ptr session; - uint64_t nextRequestId = 1; - - mutable std::mutex flowControlMutex; - bool flowControlEnabled = false; - int64_t flowCreditFrames = 0; - int64_t flowCreditBytes = 0; - - void setFlowControlEnabled(bool enabled) - { - std::lock_guard lock(flowControlMutex); - if (enabled) { - if (!flowControlEnabled) { - flowControlEnabled = true; - flowCreditFrames = kFlowCreditMaxFrames; - flowCreditBytes = kFlowCreditMaxBytes; - } - return; - } - flowControlEnabled = false; - flowCreditFrames = 0; - flowCreditBytes = 0; - } - - [[nodiscard]] std::pair grantFlowCredits(int64_t frames, int64_t bytes) - { - std::lock_guard lock(flowControlMutex); - if (!flowControlEnabled) { - return {0, 0}; - } - const auto safeFrames = std::max(0, frames); - const auto safeBytes = std::max(0, bytes); - const auto oldFrames = flowCreditFrames; - const auto oldBytes = flowCreditBytes; - flowCreditFrames = std::min(kFlowCreditMaxFrames, flowCreditFrames + safeFrames); - flowCreditBytes = std::min(kFlowCreditMaxBytes, flowCreditBytes + safeBytes); - return {flowCreditFrames - oldFrames, flowCreditBytes - oldBytes}; - } - - [[nodiscard]] bool consumeFlowCreditForFrame(int64_t frameSizeBytes) - { - std::lock_guard lock(flowControlMutex); - if (!flowControlEnabled) { - return true; - } - if (flowCreditFrames <= 0 || flowCreditBytes <= 0) { - return false; - } - flowCreditFrames -= 1; - flowCreditBytes = std::max(0, flowCreditBytes - std::max(0, frameSizeBytes)); - return true; - } + key.layer_ = REQUEST_TILE_LAYER_TYPE; + return key; +} - [[nodiscard]] FlowControlStateSnapshot flowControlSnapshot() const - { - std::lock_guard lock(flowControlMutex); - return FlowControlStateSnapshot{ - .enabled = flowControlEnabled, - .creditFrames = flowCreditFrames, - .creditBytes = flowCreditBytes, - }; - } +/// Snapshot of flow-control state exposed to `/status-data`. +struct FlowControlStateSnapshot +{ + bool enabled = false; + int64_t creditFrames = 0; }; class TilesWsSession : public std::enable_shared_from_this { public: + /// Construct one websocket session object bound 1:1 to a websocket connection. TilesWsSession( HttpService& service, std::weak_ptr conn, - std::weak_ptr connState, - uint64_t requestId, - AuthHeaders authHeaders, - TileLayerStream::StringPoolOffsetMap initialOffsets) + AuthHeaders authHeaders) : service_(service), conn_(std::move(conn)), - connState_(std::move(connState)), - requestId_(requestId), authHeaders_(std::move(authHeaders)), - offsets_(std::move(initialOffsets)), writer_( std::make_unique( [this](std::string msg, TileLayerStream::MessageType type) { onWriterMessage(std::move(msg), type); }, - offsets_)) + writerOffsets_)) { gTilesWsMetrics.activeSessions.fetch_add(1, std::memory_order_relaxed); } + /// Destroy the session and abort any in-flight backend work. ~TilesWsSession() { gTilesWsMetrics.activeSessions.fetch_sub(1, std::memory_order_relaxed); @@ -255,12 +212,14 @@ class TilesWsSession : public std::enable_shared_from_this TilesWsSession(TilesWsSession const&) = delete; TilesWsSession& operator=(TilesWsSession const&) = delete; + /// Register this session in the global weak list used for `/status-data` snapshots. void registerForMetrics() { std::lock_guard lock(gTrackedSessionsMutex); gTrackedSessions.push_back(weak_from_this()); } + /// Return currently queued controller frames/bytes. [[nodiscard]] std::pair pendingSnapshot() { std::lock_guard lock(mutex_); @@ -272,26 +231,124 @@ class TilesWsSession : public std::enable_shared_from_this return {pendingFrames, pendingBytes}; } - void onFlowGrant() + /// Return flow-control state for `/status-data` metrics. + [[nodiscard]] FlowControlStateSnapshot flowControlSnapshot() const + { + std::lock_guard lock(flowControlMutex_); + return FlowControlStateSnapshot{ + .enabled = flowControlEnabled_, + .creditFrames = flowCreditFrames_, + }; + } + + /// Enable/disable frame-credit flow control for this connection. + void setFlowControlEnabled(bool enabled) { + std::lock_guard lock(flowControlMutex_); + if (enabled) { + if (!flowControlEnabled_) { + flowControlEnabled_ = true; + flowCreditFrames_ = FLOW_CREDIT_MAX_FRAMES; + } + return; + } + flowControlEnabled_ = false; + flowCreditFrames_ = 0; + } + + /// Add frame credits granted by the client and return credits actually applied. + [[nodiscard]] int64_t grantFlowCredits(int64_t frames) + { + std::lock_guard lock(flowControlMutex_); + if (!flowControlEnabled_) { + return 0; + } + const auto safeFrames = std::max(0, frames); + const auto oldFrames = flowCreditFrames_; + flowCreditFrames_ = std::min(FLOW_CREDIT_MAX_FRAMES, flowCreditFrames_ + safeFrames); + return flowCreditFrames_ - oldFrames; + } + + /// Patch per-connection string-pool offsets supplied by the client request. + [[nodiscard]] bool applyStringPoolOffsetsPatch(const nlohmann::json& offsetsJson, std::string& errorMessage) + { + if (!offsetsJson.is_object()) { + errorMessage = "stringPoolOffsets must be an object."; + return false; + } + + try { + std::lock_guard lock(mutex_); + for (auto const& item : offsetsJson.items()) { + const auto value = item.value().get(); + committedStringPoolOffsets_[item.key()] = value; + writerOffsets_[item.key()] = value; + } + return true; + } + catch (const std::exception& e) { + errorMessage = fmt::format("Invalid stringPoolOffsets: {}", e.what()); + return false; + } + } + + /// Allocate a request id while respecting optional client-provided request ids. + [[nodiscard]] uint64_t allocateRequestId(const nlohmann::json& requestJson) + { + uint64_t requestId = nextRequestId_++; + if (auto requestIdIt = requestJson.find("requestId"); + requestIdIt != requestJson.end() + && (requestIdIt->is_number_integer() || requestIdIt->is_number_unsigned())) { + const auto parsedRequestId = parseNonNegativeInt64(requestJson, "requestId"); + if (parsedRequestId > 0) { + requestId = static_cast(parsedRequestId); + nextRequestId_ = std::max(nextRequestId_, requestId + 1); + } + } + return requestId; + } + + /// Consume granted sent-frame slots and restart draining. + void onFlowGrant(int64_t grantedFrames) + { + if (grantedFrames > 0) { + consumeSentFlowFrames(grantedFrames); + } scheduleDrain(); } - void start(const nlohmann::json& j) + /// Parse and apply a full logical tile request update from the client. + void updateFromClientRequest(const nlohmann::json& j, uint64_t requestId) { auto requestsIt = j.find("requests"); if (requestsIt == j.end() || !requestsIt->is_array()) { + // Invalid request payload: publish an immediate status error for observability. + { + std::lock_guard lock(mutex_); + requestId_ = requestId; + requestInfos_.clear(); + requestStatuses_.clear(); + statusEmissionEnabled_ = true; + } + queueRequestContextMessage(); queueStatusMessage("Missing or invalid 'requests' array"); scheduleDrain(); return; } - try { - requests_.clear(); - requests_.reserve(requestsIt->size()); - requestStatuses_.clear(); - requestStatuses_.reserve(requestsIt->size()); + struct ParsedRequest + { + std::string mapId; + std::string layerId; + std::vector tileIds; + }; + std::vector parsedRequests; + std::set desiredTileKeys; + std::map nextTilePriorityRanks; + int64_t nextPriorityRank = 0; + try { + parsedRequests.reserve(requestsIt->size()); for (auto const& requestJson : *requestsIt) { const std::string mapId = requestJson.at("mapId").get(); const std::string layerId = requestJson.at("layerId").get(); @@ -303,58 +360,128 @@ class TilesWsSession : public std::enable_shared_from_this std::vector tileIds; tileIds.reserve(tileIdsJson.size()); for (auto const& tid : tileIdsJson) { - tileIds.emplace_back(tid.get()); + const auto tileId = TileId{tid.get()}; + tileIds.emplace_back(tileId); + const auto tileKey = makeCanonicalRequestedTileKey(mapId, layerId, tileId); + desiredTileKeys.insert(tileKey); + if (nextTilePriorityRanks.find(tileKey) == nextTilePriorityRanks.end()) { + nextTilePriorityRanks.emplace(tileKey, nextPriorityRank++); + } } - requests_.push_back(std::make_shared(mapId, layerId, std::move(tileIds))); - requestStatuses_.push_back(RequestStatus::Open); + parsedRequests.push_back(ParsedRequest{ + .mapId = mapId, + .layerId = layerId, + .tileIds = std::move(tileIds), + }); } } catch (const std::exception& e) { + { + std::lock_guard lock(mutex_); + requestId_ = requestId; + requestInfos_.clear(); + requestStatuses_.clear(); + statusEmissionEnabled_ = true; + } + queueRequestContextMessage(); queueStatusMessage(fmt::format("Invalid request JSON: {}", e.what())); scheduleDrain(); return; } - // Hook request callbacks before calling service_.request so early - // failures (NoDataSource/Unauthorized) still produce status updates. - const auto weak = weak_from_this(); - for (size_t i = 0; i < requests_.size(); ++i) { - auto& req = requests_[i]; - req->onFeatureLayer([weak](auto&& layer) { - if (auto self = weak.lock()) { - self->onTileLayer(std::forward(layer)); - } + std::vector serviceRequests; + std::vector nextRequestStatuses(parsedRequests.size(), RequestStatus::Success); + std::vector nextRequestInfos; + nextRequestInfos.reserve(parsedRequests.size()); + + for (size_t index = 0; index < parsedRequests.size(); ++index) { + auto& parsed = parsedRequests[index]; + nextRequestInfos.push_back(RequestInfo{ + .mapId = parsed.mapId, + .layerId = parsed.layerId, }); - req->onSourceDataLayer([weak](auto&& layer) { + + std::vector tileIdsToFetch; + tileIdsToFetch.reserve(parsed.tileIds.size()); + { + std::lock_guard lock(mutex_); + for (const auto& tileId : parsed.tileIds) { + const auto requestedTileKey = makeCanonicalRequestedTileKey(parsed.mapId, parsed.layerId, tileId); + const bool alreadyQueued = + queuedTileFrameRefCount_.find(requestedTileKey) != queuedTileFrameRefCount_.end(); + const bool alreadySentNotGranted = + sentTileFrameRefCount_.find(requestedTileKey) != sentTileFrameRefCount_.end(); + // Skip backend fetches for tiles already queued or already sent but not yet granted. + if (!alreadyQueued && !alreadySentNotGranted) { + tileIdsToFetch.push_back(tileId); + } + } + } + if (tileIdsToFetch.empty()) { + continue; + } + + auto request = std::make_shared( + parsed.mapId, + parsed.layerId, + std::move(tileIdsToFetch)); + serviceRequests.push_back(request); + { + std::lock_guard lock(mutex_); + activeRequests_.push_back(request); + } + nextRequestStatuses[index] = RequestStatus::Open; + + const auto weak = weak_from_this(); + const auto expectedRequestId = requestId; + request->onFeatureLayer([weak](auto&& layer) { if (auto self = weak.lock()) { self->onTileLayer(std::forward(layer)); } }); - req->onLayerLoadStateChanged([weak](MapTileKey const& key, TileLayer::LoadState state) { + request->onSourceDataLayer([weak](auto&& layer) { if (auto self = weak.lock()) { - self->onLoadStateChanged(key, state); + self->onTileLayer(std::forward(layer)); } }); - req->onDone_ = [weak, i](RequestStatus status) { + if (EMIT_LOAD_STATE_FRAMES) { + request->onLayerLoadStateChanged([weak](MapTileKey const& key, TileLayer::LoadState state) { + if (auto self = weak.lock()) { + self->onLoadStateChanged(key, state); + } + }); + } + request->onDone_ = [weak, index, expectedRequestId, request](RequestStatus status) { if (auto self = weak.lock()) { - self->onRequestDone(i, status); + self->onRequestDone(index, expectedRequestId, request, status); } }; } - // Start processing (may synchronously set request statuses). - queueRequestContextMessage(); - (void)service_.request(requests_, authHeaders_); - { std::lock_guard lock(mutex_); + requestId_ = requestId; + requestInfos_ = std::move(nextRequestInfos); + requestStatuses_ = std::move(nextRequestStatuses); + desiredTileKeys_ = std::move(desiredTileKeys); + tilePriorityRanks_ = std::move(nextTilePriorityRanks); + // When request scope shrinks, remove stale tile data already queued for send. + filterOutgoingByDesiredLocked(); + // Refresh ordering so queued tiles follow the latest request priority. + reprioritizeOutgoingLocked(); statusEmissionEnabled_ = true; } + + queueRequestContextMessage(); + if (!serviceRequests.empty()) { + (void)service_.request(serviceRequests, authHeaders_); + } queueStatusMessage({}); scheduleDrain(); } + /// Cancel current requests, clear queued frames, and emit a terminal status. void cancel(std::string reason) { cancelled_ = true; @@ -366,18 +493,19 @@ class TilesWsSession : public std::enable_shared_from_this } // Abort in-flight requests (best-effort). - for (auto const& r : requests_) { + for (auto const& r : activeRequests_) { if (!r || r->isDone()) continue; service_.abort(r); } + activeRequests_.clear(); // Refresh locally cached statuses after aborting. { std::lock_guard lock(mutex_); - for (size_t i = 0; i < requests_.size() && i < requestStatuses_.size(); ++i) { - if (requests_[i]) { - requestStatuses_[i] = requests_[i]->getStatus(); + for (auto& status : requestStatuses_) { + if (status == RequestStatus::Open) { + status = RequestStatus::Aborted; } } } @@ -387,27 +515,221 @@ class TilesWsSession : public std::enable_shared_from_this } private: + /// Consume exactly one frame credit before sending a flow-controlled frame. + [[nodiscard]] bool consumeFlowCreditForFrame() + { + std::lock_guard lock(flowControlMutex_); + if (!flowControlEnabled_) { + return true; + } + if (flowCreditFrames_ <= 0) { + return false; + } + flowCreditFrames_ -= 1; + return true; + } + + /// Lightweight metadata emitted in status payloads for each logical request. + struct RequestInfo + { + std::string mapId; + std::string layerId; + }; + + /// One queued websocket frame plus metadata used for bookkeeping. struct OutgoingFrame { std::string bytes; TileLayerStream::MessageType type{TileLayerStream::MessageType::None}; std::optional> stringPoolCommit; + std::optional requestedTileKey; + int64_t priorityRank = LOWEST_TILE_PRIORITY; }; + /// Batched writer output captured while serializing one tile layer. struct WriterMessage { std::string bytes; TileLayerStream::MessageType type{TileLayerStream::MessageType::None}; }; + /// Increment queued/sent reference counters for one canonical tile key. + void incrementFrameRefCount(std::map& counts, const MapTileKey& key) + { + counts[key] += 1; + } + + /// Decrement queued/sent reference counters and erase exhausted entries. + void decrementFrameRefCount(std::map& counts, const MapTileKey& key) + { + auto it = counts.find(key); + if (it == counts.end()) { + return; + } + it->second -= 1; + if (it->second <= 0) { + counts.erase(it); + } + } + + /// Mark a frame as queued so request updates can avoid duplicate backend fetches. + void trackQueuedFrameLocked(const OutgoingFrame& frame) + { + if (frame.requestedTileKey) { + incrementFrameRefCount(queuedTileFrameRefCount_, *frame.requestedTileKey); + } + } + + /// Remove a frame from queued bookkeeping once it is dequeued or dropped. + void untrackQueuedFrameLocked(const OutgoingFrame& frame) + { + if (frame.requestedTileKey) { + decrementFrameRefCount(queuedTileFrameRefCount_, *frame.requestedTileKey); + } + } + + /// Track flow-controlled frames that were sent but not yet granted back by the client. + void trackSentFrameLocked(const OutgoingFrame& frame) + { + sentFlowFrames_.push_back(frame.requestedTileKey); + if (frame.requestedTileKey) { + incrementFrameRefCount(sentTileFrameRefCount_, *frame.requestedTileKey); + } + } + + /// Apply client grants to the sent-frame ledger to release in-flight dedupe entries. + void consumeSentFlowFrames(int64_t grantedFrames) + { + std::lock_guard lock(mutex_); + for (int64_t i = 0; i < grantedFrames && !sentFlowFrames_.empty(); ++i) { + auto key = std::move(sentFlowFrames_.front()); + sentFlowFrames_.pop_front(); + if (key) { + decrementFrameRefCount(sentTileFrameRefCount_, *key); + } + } + } + + /// Look up the current priority rank for one tile key, defaulting to lowest priority. + [[nodiscard]] int64_t tilePriorityRankLocked(const MapTileKey& tileKey) const + { + const auto it = tilePriorityRanks_.find(tileKey); + if (it == tilePriorityRanks_.end()) { + return LOWEST_TILE_PRIORITY; + } + return it->second; + } + + /// Refresh one queued frame's cached priority rank against the latest request priorities. + void refreshFramePriorityLocked(OutgoingFrame& frame) const + { + if (!frame.requestedTileKey) { + frame.priorityRank = LOWEST_TILE_PRIORITY; + return; + } + frame.priorityRank = tilePriorityRankLocked(*frame.requestedTileKey); + } + + /// Compare two frames for queue order; returns true if lhs should be sent before rhs. + [[nodiscard]] static bool framePrecedes(const OutgoingFrame& lhs, const OutgoingFrame& rhs) + { + const bool lhsIsStringPool = lhs.type == TileLayerStream::MessageType::StringPool; + const bool rhsIsStringPool = rhs.type == TileLayerStream::MessageType::StringPool; + if (lhsIsStringPool != rhsIsStringPool) { + // String pool updates always outrank everything else. + return lhsIsStringPool; + } + + const bool lhsHasTile = lhs.requestedTileKey.has_value(); + const bool rhsHasTile = rhs.requestedTileKey.has_value(); + if (lhsHasTile != rhsHasTile) { + // Keep non-tile control frames ahead of regular tile data frames. + return !lhsHasTile; + } + if (!lhsHasTile) { + return false; + } + return lhs.priorityRank < rhs.priorityRank; + } + + /// Drop queued tile data frames that no longer belong to the latest request set. + void filterOutgoingByDesiredLocked() + { + if (outgoing_.empty()) { + return; + } + + int64_t droppedFrames = 0; + int64_t droppedBytes = 0; + std::deque filtered; + for (auto& frame : outgoing_) { + const bool dropLoadStateFrame = !EMIT_LOAD_STATE_FRAMES + && frame.type == TileLayerStream::MessageType::LoadStateChange; + const bool dropStaleTileFrame = frame.requestedTileKey + && desiredTileKeys_.find(*frame.requestedTileKey) == desiredTileKeys_.end(); + const bool dropFrame = dropLoadStateFrame || dropStaleTileFrame; + if (dropFrame) { + ++droppedFrames; + droppedBytes += static_cast(frame.bytes.size()); + untrackQueuedFrameLocked(frame); + continue; + } + filtered.push_back(std::move(frame)); + } + outgoing_ = std::move(filtered); + if (droppedFrames > 0) { + gTilesWsMetrics.totalDroppedFrames.fetch_add(droppedFrames, std::memory_order_relaxed); + gTilesWsMetrics.totalDroppedBytes.fetch_add(droppedBytes, std::memory_order_relaxed); + } + } + + /// Reorder queued frames according to string-pool and tile-priority policy. + void reprioritizeOutgoingLocked() + { + if (outgoing_.size() < 2) { + return; + } + + for (auto& frame : outgoing_) { + refreshFramePriorityLocked(frame); + } + + std::vector reordered; + reordered.reserve(outgoing_.size()); + for (auto& frame : outgoing_) { + reordered.push_back(std::move(frame)); + } + + std::stable_sort( + reordered.begin(), + reordered.end(), + [](const OutgoingFrame& lhs, const OutgoingFrame& rhs) { return framePrecedes(lhs, rhs); }); + + outgoing_.clear(); + for (auto& frame : reordered) { + outgoing_.push_back(std::move(frame)); + } + } + + /// Append one frame to the websocket controller queue and update counters. void enqueueOutgoingLocked(OutgoingFrame&& frame) { + refreshFramePriorityLocked(frame); + trackQueuedFrameLocked(frame); const auto bytes = static_cast(frame.bytes.size()); - outgoing_.push_back(std::move(frame)); + auto insertIt = outgoing_.end(); + for (auto it = outgoing_.begin(); it != outgoing_.end(); ++it) { + if (framePrecedes(frame, *it)) { + insertIt = it; + break; + } + } + outgoing_.insert(insertIt, std::move(frame)); gTilesWsMetrics.totalQueuedFrames.fetch_add(1, std::memory_order_relaxed); gTilesWsMetrics.totalQueuedBytes.fetch_add(bytes, std::memory_order_relaxed); } + /// Drop all queued frames and account them as controller-side drops. void clearOutgoingLocked() { if (outgoing_.empty()) { @@ -419,6 +741,7 @@ class TilesWsSession : public std::enable_shared_from_this for (auto const& frame : outgoing_) { ++droppedFrames; droppedBytes += static_cast(frame.bytes.size()); + untrackQueuedFrameLocked(frame); } outgoing_.clear(); @@ -426,6 +749,7 @@ class TilesWsSession : public std::enable_shared_from_this gTilesWsMetrics.totalDroppedBytes.fetch_add(droppedBytes, std::memory_order_relaxed); } + /// Internal cancel path used by destructor/connection tear-down (no status emission). void cancelNoStatus() { if (cancelled_.exchange(true)) @@ -437,13 +761,15 @@ class TilesWsSession : public std::enable_shared_from_this clearOutgoingLocked(); } - for (auto const& r : requests_) { + for (auto const& r : activeRequests_) { if (!r || r->isDone()) continue; service_.abort(r); } + activeRequests_.clear(); } + /// Collect writer callbacks generated while serializing one tile layer. void onWriterMessage(std::string msg, TileLayerStream::MessageType type) { // Writer messages are only generated from within onTileLayer under mutex_. @@ -453,6 +779,7 @@ class TilesWsSession : public std::enable_shared_from_this currentWriteBatch_->push_back(WriterMessage{std::move(msg), type}); } + /// Convert one backend tile layer into outgoing websocket frames. void onTileLayer(TileLayer::Ptr const& layer) { if (cancelled_) @@ -460,12 +787,17 @@ class TilesWsSession : public std::enable_shared_from_this if (!layer) return; + const auto requestedTileKey = makeCanonicalRequestedTileKey(layer->id()); std::optional> stringPoolCommit; { std::lock_guard lock(mutex_); if (cancelled_) return; + // Late-arriving tile for an outdated request: drop before serialization work. + if (desiredTileKeys_.find(requestedTileKey) == desiredTileKeys_.end()) { + return; + } if (currentWriteBatch_.has_value()) { raise("TilesWsSession writer callback re-entered"); @@ -475,11 +807,11 @@ class TilesWsSession : public std::enable_shared_from_this auto batch = std::move(*currentWriteBatch_); currentWriteBatch_.reset(); - // If a StringPool message was generated, the writer updates offsets_ + // If a StringPool message was generated, the writer updates writerOffsets_ // to the new highest string ID for this node after emitting it. const auto nodeId = layer->nodeId(); - const auto it = offsets_.find(nodeId); - if (it != offsets_.end()) { + const auto it = writerOffsets_.find(nodeId); + if (it != writerOffsets_.end()) { const auto newOffset = it->second; for (auto const& m : batch) { if (m.type == TileLayerStream::MessageType::StringPool) { @@ -495,6 +827,11 @@ class TilesWsSession : public std::enable_shared_from_this frame.type = m.type; if (m.type == TileLayerStream::MessageType::StringPool) { frame.stringPoolCommit = stringPoolCommit; + frame.requestedTileKey = requestedTileKey; + } + if (m.type == TileLayerStream::MessageType::TileFeatureLayer + || m.type == TileLayerStream::MessageType::TileSourceDataLayer) { + frame.requestedTileKey = requestedTileKey; } enqueueOutgoingLocked(std::move(frame)); } @@ -503,7 +840,12 @@ class TilesWsSession : public std::enable_shared_from_this scheduleDrain(); } - void onRequestDone(size_t requestIndex, RequestStatus status) + /// Update per-request completion state and emit status when it changes. + void onRequestDone( + size_t requestIndex, + uint64_t expectedRequestId, + const LayerTilesRequest::Ptr& completedRequest, + RequestStatus status) { if (cancelled_) return; @@ -513,12 +855,21 @@ class TilesWsSession : public std::enable_shared_from_this std::lock_guard lock(mutex_); if (cancelled_) return; - if (requestIndex >= requestStatuses_.size()) - return; - if (requestStatuses_[requestIndex] == status) - return; - requestStatuses_[requestIndex] = status; - shouldEmit = statusEmissionEnabled_; + activeRequests_.erase( + std::remove_if( + activeRequests_.begin(), + activeRequests_.end(), + [&](const LayerTilesRequest::Ptr& req) { + return !req || req == completedRequest || req->isDone(); + }), + activeRequests_.end()); + if (expectedRequestId == requestId_ && requestIndex < requestStatuses_.size()) { + if (requestStatuses_[requestIndex] == status) { + return; + } + requestStatuses_[requestIndex] = status; + shouldEmit = statusEmissionEnabled_; + } } if (shouldEmit) { @@ -527,6 +878,7 @@ class TilesWsSession : public std::enable_shared_from_this } } + /// Queue a status frame describing the current request statuses. void queueStatusMessage(std::string message) { OutgoingFrame frame; @@ -538,6 +890,7 @@ class TilesWsSession : public std::enable_shared_from_this } } + /// Queue a request-context frame so the client can track the active request id. void queueRequestContextMessage() { OutgoingFrame frame; @@ -550,10 +903,22 @@ class TilesWsSession : public std::enable_shared_from_this } } + /// Forward backend tile load-state changes for tiles still requested by the client. void onLoadStateChanged(MapTileKey const& key, TileLayer::LoadState state) { + if (!EMIT_LOAD_STATE_FRAMES) { + return; + } if (cancelled_) return; + const auto requestedTileKey = makeCanonicalRequestedTileKey(key); + { + std::lock_guard lock(mutex_); + // Keep load-state traffic scoped to the currently requested tile set. + if (desiredTileKeys_.find(requestedTileKey) == desiredTileKeys_.end()) { + return; + } + } OutgoingFrame frame; frame.bytes = encodeStreamMessage( @@ -567,6 +932,7 @@ class TilesWsSession : public std::enable_shared_from_this scheduleDrain(); } + /// Build the JSON payload for `mapget.tiles.status`. [[nodiscard]] std::string buildStatusPayload(std::string message) { nlohmann::json requestsJson = nlohmann::json::array(); @@ -574,19 +940,14 @@ class TilesWsSession : public std::enable_shared_from_this { std::lock_guard lock(mutex_); - for (size_t i = 0; i < requests_.size(); ++i) { + for (size_t i = 0; i < requestInfos_.size(); ++i) { const auto status = (i < requestStatuses_.size()) ? requestStatuses_[i] : RequestStatus::Open; allDone &= (status != RequestStatus::Open); nlohmann::json reqJson = nlohmann::json::object(); reqJson["index"] = i; - if (i < requests_.size() && requests_[i]) { - reqJson["mapId"] = requests_[i]->mapId_; - reqJson["layerId"] = requests_[i]->layerId_; - } else { - reqJson["mapId"] = ""; - reqJson["layerId"] = ""; - } + reqJson["mapId"] = requestInfos_[i].mapId; + reqJson["layerId"] = requestInfos_[i].layerId; reqJson["status"] = static_cast>(status); reqJson["statusText"] = std::string(requestStatusToString(status)); requestsJson.push_back(std::move(reqJson)); @@ -602,6 +963,7 @@ class TilesWsSession : public std::enable_shared_from_this }).dump(); } + /// Build the JSON payload for `mapget.tiles.load-state`. [[nodiscard]] std::string buildLoadStatePayload(MapTileKey const& key, TileLayer::LoadState state) const { return nlohmann::json::object({ @@ -615,6 +977,7 @@ class TilesWsSession : public std::enable_shared_from_this }).dump(); } + /// Build the JSON payload for `mapget.tiles.request-context`. [[nodiscard]] std::string buildRequestContextPayload() const { return nlohmann::json::object({ @@ -623,6 +986,7 @@ class TilesWsSession : public std::enable_shared_from_this }).dump(); } + /// Schedule queue draining while guaranteeing at most one active drainer. void scheduleDrain() { if (drainScheduled_.exchange(true)) @@ -630,6 +994,7 @@ class TilesWsSession : public std::enable_shared_from_this drainNow(); } + /// Drain queued frames to Drogon while respecting flow-control credits. void drainNow() { gTilesWsMetrics.totalDrainCalls.fetch_add(1, std::memory_order_relaxed); @@ -644,12 +1009,9 @@ class TilesWsSession : public std::enable_shared_from_this return; } - constexpr size_t maxFramesPerDrain = 64; - constexpr size_t maxBytesPerDrain = 2 * 1024 * 1024; - size_t drainedBytes = 0; bool blockedByFlowControl = false; - for (size_t i = 0; i < maxFramesPerDrain && drainedBytes < maxBytesPerDrain; ++i) { + for (size_t i = 0; i < MAX_FRAMES_PER_DRAIN; ++i) { OutgoingFrame frame; { std::lock_guard lock(mutex_); @@ -658,6 +1020,7 @@ class TilesWsSession : public std::enable_shared_from_this } frame = std::move(outgoing_.front()); outgoing_.pop_front(); + untrackQueuedFrameLocked(frame); } const auto frameBytes = static_cast(frame.bytes.size()); @@ -669,23 +1032,24 @@ class TilesWsSession : public std::enable_shared_from_this } if (isFlowControlledDataFrameType(frame.type)) { - auto state = connState_.lock(); - if (!state || !state->consumeFlowCreditForFrame(frameBytes)) { + // No credits available: put frame back at the front and stop this drain pass. + if (!consumeFlowCreditForFrame()) { std::lock_guard lock(mutex_); outgoing_.push_front(std::move(frame)); + trackQueuedFrameLocked(outgoing_.front()); blockedByFlowControl = true; break; } + std::lock_guard lock(mutex_); + trackSentFrameLocked(frame); } - drainedBytes += static_cast(frameBytes); gTilesWsMetrics.totalForwardedFrames.fetch_add(1, std::memory_order_relaxed); gTilesWsMetrics.totalForwardedBytes.fetch_add(frameBytes, std::memory_order_relaxed); conn->send(frame.bytes, drogon::WebSocketMessageType::Binary); if (frame.stringPoolCommit) { - if (auto state = connState_.lock()) { - state->stringPoolOffsets[frame.stringPoolCommit->first] = frame.stringPoolCommit->second; - } + std::lock_guard lock(mutex_); + committedStringPoolOffsets_[frame.stringPoolCommit->first] = frame.stringPoolCommit->second; } } @@ -710,19 +1074,29 @@ class TilesWsSession : public std::enable_shared_from_this HttpService& service_; std::weak_ptr conn_; - std::weak_ptr connState_; - uint64_t requestId_; + uint64_t requestId_ = 0; + uint64_t nextRequestId_ = 1; AuthHeaders authHeaders_; + mutable std::mutex flowControlMutex_; + bool flowControlEnabled_ = false; + int64_t flowCreditFrames_ = 0; + std::mutex mutex_; std::deque outgoing_; - - std::vector requests_; + std::vector requestInfos_; std::vector requestStatuses_; + std::vector activeRequests_; + std::set desiredTileKeys_; + std::map tilePriorityRanks_; + std::map queuedTileFrameRefCount_; + std::map sentTileFrameRefCount_; + std::deque> sentFlowFrames_; bool statusEmissionEnabled_ = false; - TileLayerStream::StringPoolOffsetMap offsets_; + TileLayerStream::StringPoolOffsetMap committedStringPoolOffsets_; + TileLayerStream::StringPoolOffsetMap writerOffsets_; std::unique_ptr writer_; std::optional> currentWriteBatch_; @@ -733,29 +1107,30 @@ class TilesWsSession : public std::enable_shared_from_this class TilesWebSocketController final : public drogon::WebSocketController { public: + /// Build the websocket controller bound to one shared HttpService instance. explicit TilesWebSocketController(HttpService& service) : service_(service) {} + /// Create and attach one `TilesWsSession` per accepted websocket connection. void handleNewConnection(const drogon::HttpRequestPtr& req, const drogon::WebSocketConnectionPtr& conn) override { gTilesWsMetrics.activeConnections.fetch_add(1, std::memory_order_relaxed); - auto state = std::make_shared(); - state->authHeaders = authHeadersFromRequest(req); - { - std::lock_guard lock(gTrackedConnectionsMutex); - gTrackedConnections.push_back(state); - } - conn->setContext(std::move(state)); + auto session = std::make_shared(service_, conn, authHeadersFromRequest(req)); + session->registerForMetrics(); + conn->setContext(std::move(session)); } + /// Handle control and request messages from the websocket client. void handleNewMessage( const drogon::WebSocketConnectionPtr& conn, std::string&& message, const drogon::WebSocketMessageType& type) override { - auto state = conn->getContext(); - if (!state) { - state = std::make_shared(); - conn->setContext(state); + auto session = conn->getContext(); + if (!session) { + // This is a defensive fallback for unexpected context loss. + session = std::make_shared(service_, conn, AuthHeaders{}); + session->registerForMetrics(); + conn->setContext(session); } if (type != drogon::WebSocketMessageType::Text) { @@ -789,16 +1164,11 @@ class TilesWebSocketController final : public drogon::WebSocketControllerget(); } - if (messageType == kFlowGrantType) { - auto [grantedFrames, grantedBytes] = state->grantFlowCredits( - parseNonNegativeInt64(j, "frames"), - parseNonNegativeInt64(j, "bytes")); + if (messageType == FLOW_GRANT_TYPE) { + const auto grantedFrames = session->grantFlowCredits(parseNonNegativeInt64(j, "frames")); gTilesWsMetrics.totalFlowGrantMessages.fetch_add(1, std::memory_order_relaxed); gTilesWsMetrics.totalFlowGrantFrames.fetch_add(grantedFrames, std::memory_order_relaxed); - gTilesWsMetrics.totalFlowGrantBytes.fetch_add(grantedBytes, std::memory_order_relaxed); - if (state->session) { - state->session->onFlowGrant(); - } + session->onFlowGrant(grantedFrames); return; } @@ -806,71 +1176,33 @@ class TilesWebSocketController final : public drogon::WebSocketControlleris_boolean()) { flowControl = flowControlIt->get(); } - state->setFlowControlEnabled(flowControl); + session->setFlowControlEnabled(flowControl); // Patch per-connection string pool offsets if supplied. if (j.contains("stringPoolOffsets")) { - if (!j["stringPoolOffsets"].is_object()) { - const auto payload = nlohmann::json::object({ - {"type", "mapget.tiles.status"}, - {"allDone", true}, - {"requests", nlohmann::json::array()}, - {"message", "stringPoolOffsets must be an object."}, - }).dump(); - conn->send(encodeStreamMessage(TileLayerStream::MessageType::Status, payload), drogon::WebSocketMessageType::Binary); - return; - } - try { - for (auto const& item : j["stringPoolOffsets"].items()) { - state->stringPoolOffsets[item.key()] = item.value().get(); - } - } - catch (const std::exception& e) { + std::string errorMessage; + if (!session->applyStringPoolOffsetsPatch(j["stringPoolOffsets"], errorMessage)) { const auto payload = nlohmann::json::object({ {"type", "mapget.tiles.status"}, {"allDone", true}, {"requests", nlohmann::json::array()}, - {"message", fmt::format("Invalid stringPoolOffsets: {}", e.what())}, + {"message", std::move(errorMessage)}, }).dump(); conn->send(encodeStreamMessage(TileLayerStream::MessageType::Status, payload), drogon::WebSocketMessageType::Binary); return; } } - if (state->session) { - gTilesWsMetrics.replacedRequests.fetch_add(1, std::memory_order_relaxed); - state->session->cancel("Replaced by a new /tiles WebSocket request."); - state->session.reset(); - } - - uint64_t requestId = state->nextRequestId++; - if (auto requestIdIt = j.find("requestId"); - requestIdIt != j.end() && (requestIdIt->is_number_integer() || requestIdIt->is_number_unsigned())) { - const auto parsedRequestId = parseNonNegativeInt64(j, "requestId"); - if (parsedRequestId > 0) { - requestId = static_cast(parsedRequestId); - state->nextRequestId = std::max(state->nextRequestId, requestId + 1); - } - } - - state->session = std::make_shared( - service_, - conn, - state, - requestId, - state->authHeaders, - state->stringPoolOffsets); - state->session->registerForMetrics(); - state->session->start(j); + const auto requestId = session->allocateRequestId(j); + session->updateFromClientRequest(j, requestId); } + /// Abort outstanding backend work once the websocket is closed. void handleConnectionClosed(const drogon::WebSocketConnectionPtr& conn) override { gTilesWsMetrics.activeConnections.fetch_sub(1, std::memory_order_relaxed); - if (auto state = conn->getContext()) { - if (state->session) { - state->session->cancel("WebSocket connection closed."); - } + if (auto session = conn->getContext()) { + session->cancel("WebSocket connection closed."); } } @@ -884,11 +1216,13 @@ class TilesWebSocketController final : public drogon::WebSocketController(service)); } +/// Build the websocket metrics payload consumed by `/status-data`. nlohmann::json tilesWebSocketMetricsSnapshot() { int64_t pendingControllerFrames = 0; @@ -896,38 +1230,26 @@ nlohmann::json tilesWebSocketMetricsSnapshot() int64_t flowControlEnabledConnections = 0; int64_t flowControlBlockedConnections = 0; int64_t flowControlCreditFrames = 0; - int64_t flowControlCreditBytes = 0; { std::lock_guard lock(gTrackedSessionsMutex); auto out = gTrackedSessions.begin(); for (auto it = gTrackedSessions.begin(); it != gTrackedSessions.end(); ++it) { if (auto session = it->lock()) { auto [frames, bytes] = session->pendingSnapshot(); + const auto flowSnapshot = session->flowControlSnapshot(); pendingControllerFrames += frames; pendingControllerBytes += bytes; - *out++ = *it; - } - } - gTrackedSessions.erase(out, gTrackedSessions.end()); - } - { - std::lock_guard lock(gTrackedConnectionsMutex); - auto out = gTrackedConnections.begin(); - for (auto it = gTrackedConnections.begin(); it != gTrackedConnections.end(); ++it) { - if (auto state = it->lock()) { - const auto snapshot = state->flowControlSnapshot(); - if (snapshot.enabled) { + if (flowSnapshot.enabled) { ++flowControlEnabledConnections; - flowControlCreditFrames += snapshot.creditFrames; - flowControlCreditBytes += snapshot.creditBytes; - if (snapshot.creditFrames <= 0 || snapshot.creditBytes <= 0) { + flowControlCreditFrames += flowSnapshot.creditFrames; + if (flowSnapshot.creditFrames <= 0) { ++flowControlBlockedConnections; } } *out++ = *it; } } - gTrackedConnections.erase(out, gTrackedConnections.end()); + gTrackedSessions.erase(out, gTrackedSessions.end()); } return nlohmann::json::object({ @@ -938,7 +1260,6 @@ nlohmann::json tilesWebSocketMetricsSnapshot() {"flow-control-enabled-connections", flowControlEnabledConnections}, {"flow-control-blocked-connections", flowControlBlockedConnections}, {"flow-control-credit-frames", flowControlCreditFrames}, - {"flow-control-credit-bytes", flowControlCreditBytes}, {"total-queued-frames", nonNegative(gTilesWsMetrics.totalQueuedFrames)}, {"total-queued-bytes", nonNegative(gTilesWsMetrics.totalQueuedBytes)}, {"total-forwarded-frames", nonNegative(gTilesWsMetrics.totalForwardedFrames)}, @@ -948,7 +1269,6 @@ nlohmann::json tilesWebSocketMetricsSnapshot() {"total-drain-calls", nonNegative(gTilesWsMetrics.totalDrainCalls)}, {"total-flow-grant-messages", nonNegative(gTilesWsMetrics.totalFlowGrantMessages)}, {"total-flow-grant-frames", nonNegative(gTilesWsMetrics.totalFlowGrantFrames)}, - {"total-flow-grant-bytes", nonNegative(gTilesWsMetrics.totalFlowGrantBytes)}, {"total-flow-blocked-drains", nonNegative(gTilesWsMetrics.totalFlowBlockedDrains)}, {"replaced-requests", nonNegative(gTilesWsMetrics.replacedRequests)}, });