From 9610da7e1b8e8623c99c7013876eebbb7d163ed1 Mon Sep 17 00:00:00 2001 From: rng Date: Mon, 6 Jan 2025 15:01:43 +1100 Subject: [PATCH 01/10] Init checkin --- .../server/core/service/ElasticSearch.java | 2 +- .../server/core/service/OGCApiService.java | 18 +++++++ .../ogcapi/server/core/service/Search.java | 2 +- .../aodn/ogcapi/server/features/RestApi.java | 54 ++++++++++--------- .../ogcapi/server/features/RestServices.java | 2 +- 5 files changed, 51 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 9d374c8f..714d4b67 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -438,7 +438,7 @@ protected static FieldValue toFieldValue(String s) { } @Override - public DatasetSearchResult searchDataset( + public DatasetSearchResult searchDatasetData( String collectionId, String startDate, String endDate diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java index aad2a437..596d0305 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java @@ -14,6 +14,7 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import java.math.BigDecimal; import java.util.List; import java.util.function.BiFunction; @@ -132,4 +133,21 @@ else if (datetime.contains("/") && !datetime.contains("..")) { } } } + /** + * Convert the bbox parameter to CQL + * @param bbox + * @param filter + * @return + */ + public static String processBBoxParameter(List bbox, String filter) { + String f = null; + if(bbox.size() == 4) { + // 2D + f = String.format("BBOX(%s,%s,%s,%s)", bbox.get(0), bbox.get(1), bbox.get(2), bbox.get(3)); + } + else if(bbox.size() == 6) { + // 3D + f = String.format("BBOX(%s,%s,%%,%s,%s,%s)", bbox.get(0), bbox.get(1), bbox.get(2), bbox.get(3), bbox.get(4), bbox.get(5)); + } + } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/Search.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/Search.java index dcde8713..8e759324 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/Search.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/Search.java @@ -15,7 +15,7 @@ public interface Search { ElasticSearchBase.SearchResult searchCollections(List ids, String sortBy); ElasticSearchBase.SearchResult searchAllCollections(String sortBy) throws Exception; - DatasetSearchResult searchDataset(String collectionId, String startDate, String endDate) throws Exception; + DatasetSearchResult searchDatasetData(String collectionId, String startDate, String endDate) throws Exception; ElasticSearchBase.SearchResult searchByParameters( List targets, diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java index 64d01096..445e267b 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java @@ -5,6 +5,7 @@ import au.org.aodn.ogcapi.features.model.Collections; import au.org.aodn.ogcapi.features.model.FeatureCollectionGeoJSON; import au.org.aodn.ogcapi.features.model.FeatureGeoJSON; +import au.org.aodn.ogcapi.server.core.service.OGCApiService; import io.swagger.v3.oas.annotations.Hidden; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; @@ -31,37 +32,42 @@ public ResponseEntity getFeature(String collectionId, String fea return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build(); } - @RequestMapping( - value = {"/collections/{collectionId}/items"}, - produces = {"application/geo+json", "text/html", "application/json"}, - method = {RequestMethod.GET} - ) - public ResponseEntity getFeatures( - - @PathVariable("collectionId") String collectionId, - @RequestParam(value = "start_datetime", required = false) String startDate, - @RequestParam(value = "end_datetime", required = false) String endDate, - // keep these two parameters for future usage - @RequestParam(value= "zoom", required = false) Double zoomLevel, - @RequestParam(value="bbox", required = false) List bbox - ) { - return featuresService.getSummarizedDataset(collectionId, startDate, endDate); - } - - - +// @RequestMapping( +// value = {"/collections/{collectionId}/items"}, +// produces = {"application/geo+json", "text/html", "application/json"}, +// method = {RequestMethod.GET} +// ) +// public ResponseEntity getFeatures( +// +// @PathVariable("collectionId") String collectionId, +// @RequestParam(value = "start_datetime", required = false) String startDate, +// @RequestParam(value = "end_datetime", required = false) String endDate, +// // keep these two parameters for future usage +// @RequestParam(value= "zoom", required = false) Double zoomLevel, +// @RequestParam(value="bbox", required = false) List bbox +// ) { +// return featuresService.getSummarizedDataset(collectionId, startDate, endDate); +// } /** - * Hidden because we want to have a more functional implementation + * + * @param collectionId - The collection id + * @param limit - Limit of result return + * @param bbox - Bounding box that bounds the result set. In case of multiple bounding box, you need to issue multiple query + * @param datetime - Start/end time + * @return - The data that matches the filter criteria */ - @Hidden @Override public ResponseEntity getFeatures(String collectionId, Integer limit, List bbox, String datetime) { - return null; + String filter = null; + if (datetime != null) { + filter = OGCApiService.processDatetimeParameter(datetime, null); + } + return featuresService.getSummarizedDataset(collectionId, startDate, endDate); } /** - * @Hidden effectively disable this REST point because it is common in many places and + * Hidden effectively disable this REST point because it is common in many places and * should not implement it here, @Hidden disable swagger doc too - * @return + * @return - Not implemented */ @Hidden @Override diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java index 8fa14992..fbc49257 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java @@ -47,7 +47,7 @@ public ResponseEntity getSummarizedDataset( String endDate ) { try { - var result = search.searchDataset(collectionId, startDate, endDate); + var result = search.searchDatasetData(collectionId, startDate, endDate); return ResponseEntity.ok() .body(result.getSummarizedDataset()); } catch (Exception e) { From 8339163464e25f23eee672b12e6ac83505f75663 Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 15 Jan 2025 14:18:01 +1100 Subject: [PATCH 02/10] Checkpoint --- .../aodn/ogcapi/server/common/RestApi.java | 6 +- .../server/core/mapper/StacToCollections.java | 5 +- .../core/mapper/StacToInlineResponse2002.java | 5 +- .../core/mapper/StacToTileSetWmWGS84Q.java | 4 +- .../ogcapi/server/core/model/AssetModel.java | 48 +++++ ...earchResult.java => DataSearchResult.java} | 40 +++- .../server/core/model/DatasetModel.java | 12 -- .../ogcapi/server/core/model/DatumModel.java | 18 -- .../server/core/model/StacItemModel.java | 76 ++++++++ .../model/enumeration/CQLFeatureFields.java | 134 +++++++++++++ .../core/model/enumeration/FeatureId.java | 11 ++ .../model/enumeration/FeatureProperty.java | 4 +- .../core/service/CacheNoLandGeometry.java | 2 +- .../server/core/service/ElasticSearch.java | 181 ++++++++++-------- .../core/service/ElasticSearchBase.java | 142 +++++++++++++- .../server/core/service/OGCApiService.java | 17 +- .../ogcapi/server/core/service/Search.java | 15 +- .../ogcapi/server/core/util/CommonUtils.java | 17 ++ .../aodn/ogcapi/server/features/RestApi.java | 96 ++++++++-- .../ogcapi/server/features/RestServices.java | 35 ++-- server/src/main/resources/application.yaml | 4 +- .../core/service/OGCApiServiceTest.java | 13 +- 22 files changed, 701 insertions(+), 184 deletions(-) create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/model/AssetModel.java rename server/src/main/java/au/org/aodn/ogcapi/server/core/model/{DatasetSearchResult.java => DataSearchResult.java} (56%) delete mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetModel.java delete mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatumModel.java create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/model/StacItemModel.java create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureId.java create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/util/CommonUtils.java diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/common/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/common/RestApi.java index 89a74a62..fc32b357 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/common/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/common/RestApi.java @@ -9,6 +9,7 @@ import au.org.aodn.ogcapi.features.model.Exception; import au.org.aodn.ogcapi.server.core.mapper.StacToCollections; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType; +import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFields; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFilterType; import au.org.aodn.ogcapi.server.core.model.enumeration.OGCMediaTypeMapper; import au.org.aodn.ogcapi.server.core.service.OGCApiService; @@ -154,7 +155,10 @@ public ResponseEntity getCollections( // TODO: Support other CRS. if (CQLFilterType.convert(filterLang) == CQLFilterType.CQL && CQLCrsType.convertFromUrl(crs) == CQLCrsType.EPSG4326) { if (datetime != null) { - filter = OGCApiService.processDatetimeParameter(datetime, filter); + filter = OGCApiService.processDatetimeParameter(CQLFields.temporal.name(), datetime, filter); + } + if (bbox != null) { + filter = OGCApiService.processBBoxParameter(CQLFields.geometry.name(), bbox, filter); } return commonService.getCollectionList( q, diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java index acf3fb3f..dad013c7 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java @@ -3,6 +3,7 @@ import au.org.aodn.ogcapi.features.model.Collection; import au.org.aodn.ogcapi.features.model.Collections; import au.org.aodn.ogcapi.server.core.model.ExtendedCollections; +import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; import au.org.aodn.ogcapi.server.core.service.ElasticSearch; import org.mapstruct.Mapper; import org.opengis.filter.Filter; @@ -14,13 +15,13 @@ @Service @Mapper(componentModel = "spring") -public abstract class StacToCollections implements Converter { +public abstract class StacToCollections implements Converter, Collections> { @Value("${api.host}") protected String hostname; @Override - public Collections convert(ElasticSearch.SearchResult model, Filter filter) { + public Collections convert(ElasticSearch.SearchResult model, Filter filter) { List collections = model.getCollections().parallelStream() .map(m -> getCollection(m, filter, hostname)) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToInlineResponse2002.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToInlineResponse2002.java index 6a56347a..af55aefc 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToInlineResponse2002.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToInlineResponse2002.java @@ -1,5 +1,6 @@ package au.org.aodn.ogcapi.server.core.mapper; +import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType; import au.org.aodn.ogcapi.server.core.service.ElasticSearch; import au.org.aodn.ogcapi.tile.model.InlineResponse2002; @@ -14,13 +15,13 @@ @Service @Mapper(componentModel = "spring") -public abstract class StacToInlineResponse2002 implements Converter { +public abstract class StacToInlineResponse2002 implements Converter, InlineResponse2002> { @Value("${api.host}") protected String hostname; @Override - public InlineResponse2002 convert(ElasticSearch.SearchResult model, Filter noUse) { + public InlineResponse2002 convert(ElasticSearch.SearchResult model, Filter noUse) { List items = model.getCollections().stream() .map(m -> { TileSetItem item = new TileSetItem(); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToTileSetWmWGS84Q.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToTileSetWmWGS84Q.java index 637463a7..deeaddcb 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToTileSetWmWGS84Q.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToTileSetWmWGS84Q.java @@ -12,7 +12,7 @@ @Service @Mapper(componentModel = "spring") -public abstract class StacToTileSetWmWGS84Q implements Converter { +public abstract class StacToTileSetWmWGS84Q implements Converter, TileSet> { @Value("${api.host}") protected String hostname; @@ -27,7 +27,7 @@ protected static class TileSetWorldMercatorWGS84Quad extends TileSet { } @Override - public TileSet convert(ElasticSearch.SearchResult from, Filter noUse) { + public TileSet convert(ElasticSearch.SearchResult from, Filter noUse) { TileSetWorldMercatorWGS84Quad tileSet = new TileSetWorldMercatorWGS84Quad(); tileSet.setTileMatrixSetURI("http://www.opengis.net/def/tilematrixset/OGC/1.0/WorldMercatorWGS84Quad"); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/AssetModel.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/AssetModel.java new file mode 100644 index 00000000..1aa6a6fd --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/AssetModel.java @@ -0,0 +1,48 @@ +package au.org.aodn.ogcapi.server.core.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +@JsonInclude(JsonInclude.Include.NON_NULL) +public class AssetModel { + // https://github.com/radiantearth/stac-spec/blob/master/best-practices.md#list-of-asset-roles + public enum Role { + DATA("data"), + METADATA("metadata"), + THUMBNAIL("thumbnail"), + OVERVIEW("overview"), + VISUAL("visual"), + DATE("date"), + GRAPHIC("graphic"), + DATA_MASK("data-mask"), + SNOW_ICE("snow-ice"), + LAND_WATER("land-water"), + WATER_MASK("water-mask"), + ISO_19115("iso-19115"); + + private final String role; + + Role(String role) { + this.role = role; + } + + @Override + public String toString() { + return role; + } + } + /** + * REQUIRED. URI to the asset object. Relative and absolute URI are both allowed. Trailing slashes are significant. + */ + protected String href; + protected String title; + protected String description; + protected String type; + /** + * The semantic roles of the asset, similar to the use of rel in links. + */ + protected Role role; +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetSearchResult.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DataSearchResult.java similarity index 56% rename from server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetSearchResult.java rename to server/src/main/java/au/org/aodn/ogcapi/server/core/model/DataSearchResult.java index 37e0a1c0..0a7c93b7 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetSearchResult.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DataSearchResult.java @@ -9,13 +9,16 @@ import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; -public class DatasetSearchResult { +import static au.org.aodn.ogcapi.server.core.util.CommonUtils.safeGet; + +public class DataSearchResult { private final FeatureCollectionGeoJSON dataset; - public DatasetSearchResult() { + public DataSearchResult() { this.dataset = new FeatureCollectionGeoJSON(); initDataset(); } @@ -31,31 +34,48 @@ public FeatureCollectionGeoJSON getSummarizedDataset() { return summarizer.getSummarizedDataset(); } - public void addDatum(DatumModel datum) { + public void addDatum(StacItemModel datum) { if (datum == null) { throw new IllegalArgumentException("Datum cannot be null"); } - var feature = new FeatureGeoJSON(); + FeatureGeoJSON feature = new FeatureGeoJSON(); feature.setType(FeatureGeoJSON.TypeEnum.FEATURE); + var geometry = new PointGeoJSON(); geometry.setType(PointGeoJSON.TypeEnum.POINT); - var coordinates = new ArrayList(); + + List coordinates = new ArrayList<>(); // Don't use null checks here because it is a list and even if it is null, // it still needs "null" to occupy the space - coordinates.add(datum.getLongitude()); - coordinates.add(datum.getLatitude()); + safeGet(() -> ((Map)datum.getGeometry().get("geometry")).get("coordinates")) + .filter(item -> item instanceof List) + .map(item -> (List)item) + .ifPresent(item -> { + if(item.size() == 2) { + coordinates.add(new BigDecimal(item.get(0).toString())); + coordinates.add(new BigDecimal(item.get(1).toString())); + } + }); + geometry.setCoordinates(coordinates); feature.setGeometry(geometry); // Please add more properties if needed Map properties = new HashMap<>(); - properties.put(FeatureProperty.TIME.getValue(), datum.getTime()); - properties.put(FeatureProperty.DEPTH.getValue(), datum.getDepth()); - properties.put(FeatureProperty.COUNT.getValue(), datum.getCount()); + + safeGet(() -> ((Map)datum.getGeometry().get("properties")).get("depth")) + .ifPresent(val -> properties.put(FeatureProperty.DEPTH.getValue(), val)); + + safeGet(() -> (datum.getProperties().get("time"))) + .ifPresent(val -> properties.put(FeatureProperty.TIME.getValue(), val)); + + safeGet(() -> (datum.getProperties().get("count"))) + .ifPresent(val -> properties.put(FeatureProperty.COUNT.getValue(), val)); + feature.setProperties(properties); dataset.getFeatures().add(feature); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetModel.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetModel.java deleted file mode 100644 index 7aa62d63..00000000 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetModel.java +++ /dev/null @@ -1,12 +0,0 @@ -package au.org.aodn.ogcapi.server.core.model; - -import lombok.Builder; -import lombok.Data; - -import java.util.List; -@Data -@Builder -public class DatasetModel { - private String uuid; - private List data; -} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatumModel.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatumModel.java deleted file mode 100644 index 62be0a23..00000000 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatumModel.java +++ /dev/null @@ -1,18 +0,0 @@ -package au.org.aodn.ogcapi.server.core.model; - -import lombok.Builder; -import lombok.Data; - -import java.math.BigDecimal; - -@Data -@Builder -public class DatumModel { - - private String time; - private BigDecimal latitude; - private BigDecimal longitude; - private BigDecimal depth; - - private long count; -} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/StacItemModel.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/StacItemModel.java new file mode 100644 index 00000000..ebc19375 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/StacItemModel.java @@ -0,0 +1,76 @@ +package au.org.aodn.ogcapi.server.core.model; + +import au.org.aodn.ogcapi.features.model.FeatureCollectionGeoJSON; +import au.org.aodn.ogcapi.features.model.FeatureGeoJSON; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class StacItemModel { + + @JsonProperty("type") + protected String type; + /** + * REQUIRED. Provider identifier. The ID should be unique within the Collection that contains the Item. + */ + @JsonProperty("id") + protected String uuid; + /** + * REQUIRED. Defines the full footprint of the asset represented by this item, formatted according to RFC 7946, + * section 3.1 if a geometry is provided or section 3.2 if no geometry is provided. + * Use to generate the vector tile, the STAC format is not optimized and hard to work with for Elastic search + */ + @JsonProperty("geometry") + protected Map geometry; + /** + * REQUIRED if geometry is not null, prohibited if geometry is null. Bounding Box of the asset represented by + * this Item, formatted according to RFC 7946, section 5. + */ + @JsonProperty("bbox") + protected List> bbox; + /** + * REQUIRED. A dictionary of additional metadata for the Item. + */ + @JsonProperty("properties") + protected Map properties; + /** + * REQUIRED. List of link objects to resources and related URLs. See the best practices for details on when the + * use self links is strongly recommended. + */ + protected List links; + + protected Map assets; + /** + * The id of the STAC Collection this Item references to. This field is required if a link with a collection relation type is present and is not allowed otherwise. + */ + @JsonProperty("collection") + protected String collection; + + @JsonProperty("stac_version") + protected String stacVersion; + + @JsonProperty("stac_extensions") + public String[] getStacExtension() { + return new String[] { + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + }; + } + +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java new file mode 100644 index 00000000..a547b3ad --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java @@ -0,0 +1,134 @@ +package au.org.aodn.ogcapi.server.core.model.enumeration; + +import co.elastic.clients.elasticsearch._types.SortOptions; +import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch._types.TopLeftBottomRightGeoBounds; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.util.ObjectBuilder; +import lombok.Getter; + +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Field name for cloud optimized data index + */ +public enum CQLFeatureFields implements CQLFieldsInterface { + + id( + "id", + "id", + null, + (order) -> new SortOptions.Builder().field(f -> f.field("id.keyword").order(order)) + ), + collection( + "collection", + "collection", + null, + null + ), + temporal( + "properties.time", + "properties.time", + null, + null + ), + count( + "properties.count", + "properties.count", + null, + null + ), + geometry( + "geometry", + "geometry", + null, + null + ); + + // Field that use to do sort, elastic search treat FieldData (searchField) differently, a searchField is not + // efficient for sorting. + public final String searchField; // Field in STAC object + + @Getter + private final List displayField; + + // null value indicate it cannot be sort by that field, elastic schema change need to add keyword field in order to + // do search + @Getter + private final Function> sortBuilder; + + // We provided a default match query but there are cases where it isn't enough and need more complex + // match, one example is multiple field. Move this logic out of the parser make it easier to read + @Getter + private final Function overridePropertyEqualsToQuery; + + + CQLFeatureFields(String fields, + String displayField, + Function overridePropertyEqualsToQuery, + Function> sortBuilder) { + + this(fields, List.of(displayField), overridePropertyEqualsToQuery, sortBuilder); + } + + CQLFeatureFields(String fields, + List displayField, + Function overridePropertyEqualsToQuery, + Function> sortBuilder) { + + this.searchField = fields; + this.displayField = displayField; + this.overridePropertyEqualsToQuery = overridePropertyEqualsToQuery; + this.sortBuilder = sortBuilder; + } + + @Override + public Query getPropertyEqualToQuery(String literal) { + return null; + } + + @Override + public Query getIntersectsQuery(String literal) { + return null; + } + + @Override + public Query getIsNullQuery() { + return null; + } + + @Override + public Query getLikeQuery(String literal) { + return null; + } + + @Override + public Query getPropertyGreaterThanOrEqualsToQuery(String literal) { + return null; + } + + @Override + public Query getBoundingBoxQuery(TopLeftBottomRightGeoBounds tlbr) { + return null; + } + /** + * Given param, find any of those is not a valid CQLCollectionsField + * @param args - + * @return Invalid enum + */ + public static List findInvalidEnum(List args) { + return args.stream() + .filter(str -> { + try { + CQLFeatureFields.valueOf(str); + return false; + } + catch (IllegalArgumentException e) { + return true; + } + }) + .collect(Collectors.toList()); + } +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureId.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureId.java new file mode 100644 index 00000000..0af82b41 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureId.java @@ -0,0 +1,11 @@ +package au.org.aodn.ogcapi.server.core.model.enumeration; + +public enum FeatureId { + summary("summary"); + + private final String featureId; + + FeatureId(String id) { + this.featureId = id; + } +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureProperty.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureProperty.java index 441cf56e..7c4ba94c 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureProperty.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureProperty.java @@ -8,10 +8,10 @@ public enum FeatureProperty { DEPTH("depth"), COUNT("count"), UUID("uuid"), + GEOMETRY("geometry"), START_TIME("startTime"), END_TIME("endTime"), - COORDINATE_ACCURACY("coordinateAccuracy"), - ; + COORDINATE_ACCURACY("coordinateAccuracy"); private final String value; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java index 6d1a697c..65d4b3d5 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java @@ -35,7 +35,7 @@ public class CacheNoLandGeometry { @Scheduled(initialDelay = 1000, fixedDelay = Long.MAX_VALUE) @Cacheable("all_noland_geometry") public Map getAllNoLandGeometry() { - ElasticSearchBase.SearchResult result = elasticSearch.searchCollectionBy( + ElasticSearchBase.SearchResult result = elasticSearch.searchCollectionBy( null, null, null, diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 714d4b67..83221b24 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -1,7 +1,8 @@ package au.org.aodn.ogcapi.server.core.service; -import au.org.aodn.ogcapi.server.core.model.DatasetModel; -import au.org.aodn.ogcapi.server.core.model.DatasetSearchResult; +import au.org.aodn.ogcapi.server.core.model.DataSearchResult; +import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; +import au.org.aodn.ogcapi.server.core.model.StacItemModel; import au.org.aodn.ogcapi.server.core.model.dto.SearchSuggestionsDto; import au.org.aodn.ogcapi.server.core.model.enumeration.*; import au.org.aodn.ogcapi.server.core.parser.elastic.CQLToElasticFilterFactory; @@ -10,16 +11,20 @@ import co.elastic.clients.elasticsearch._types.FieldValue; import co.elastic.clients.elasticsearch._types.SortOptions; import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch._types.aggregations.*; import co.elastic.clients.elasticsearch._types.query_dsl.*; import co.elastic.clients.elasticsearch.core.SearchMvtRequest; import co.elastic.clients.elasticsearch.core.SearchRequest; import co.elastic.clients.elasticsearch.core.SearchResponse; import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.core.search_mvt.GridType; +import co.elastic.clients.json.JsonData; import co.elastic.clients.transport.endpoints.BinaryResponse; import co.elastic.clients.util.ObjectBuilder; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import jakarta.json.JsonValue; import lombok.extern.slf4j.Slf4j; import org.geotools.filter.text.commons.CompilerUtil; import org.geotools.filter.text.commons.Language; @@ -52,8 +57,8 @@ public class ElasticSearch extends ElasticSearchBase implements Search { @Value("${elasticsearch.vocabs_index.name}") protected String vocabsIndexName; - @Value("${elasticsearch.dataset_index.name}") - protected String datasetIndexName; + @Value("${elasticsearch.cloud_optimized_index.name}") + protected String dataIndexName; @Value("${elasticsearch.search_after.split_regex:\\|\\|}") protected String searchAfterSplitRegex; @@ -147,9 +152,9 @@ this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperatu public ResponseEntity> getAutocompleteSuggestions(String input, String cql, CQLCrsType coor) throws IOException, CQLException { Map> searchSuggestions = new HashMap<>(); - + List> suggestion = this.getSuggestionsByField(input, cql, coor); // extract parameter vocab suggestions - Set parameterVocabSuggestions = this.getSuggestionsByField(input, cql, coor) + Set parameterVocabSuggestions = suggestion .stream() .filter(item -> item.source() != null && item.source().getParameterVocabs() != null && !item.source().getParameterVocabs().isEmpty()) .flatMap(item -> item.source().getParameterVocabs().stream()) @@ -157,7 +162,7 @@ this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperatu .collect(Collectors.toSet()); searchSuggestions.put("suggested_parameter_vocabs", parameterVocabSuggestions); - Set platformVocabSuggestions = this.getSuggestionsByField(input, cql, coor) + Set platformVocabSuggestions = suggestion .stream() .filter(item -> item.source() != null && item.source().getPlatformVocabs() != null && !item.source().getPlatformVocabs().isEmpty()) .flatMap(item -> item.source().getPlatformVocabs().stream()) @@ -165,7 +170,7 @@ this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperatu .collect(Collectors.toSet()); searchSuggestions.put("suggested_platform_vocabs", platformVocabSuggestions); - Set organisationVocabSuggestions = this.getSuggestionsByField(input, cql, coor) + Set organisationVocabSuggestions = suggestion .stream() .filter(item -> item.source() != null && item.source().getOrganisationVocabs() != null && !item.source().getOrganisationVocabs().isEmpty()) .flatMap(item -> item.source().getOrganisationVocabs().stream()) @@ -174,7 +179,7 @@ this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperatu searchSuggestions.put("suggested_organisation_vocabs", organisationVocabSuggestions); // extract abstract phrases suggestions - Set abstractPhrases = this.getSuggestionsByField(input, cql, coor) + Set abstractPhrases = suggestion .stream() .filter(item -> item.source() != null && item.source().getAbstractPhrases() != null && !item.source().getAbstractPhrases().isEmpty()) .flatMap(item -> item.source().getAbstractPhrases().stream()) @@ -185,7 +190,7 @@ this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperatu return new ResponseEntity<>(searchSuggestions, HttpStatus.OK); } - protected ElasticSearchBase.SearchResult searchCollectionsByIds(List ids, Boolean isWithGeometry, String sortBy) { + protected ElasticSearchBase.SearchResult searchCollectionsByIds(List ids, Boolean isWithGeometry, String sortBy) { List queries = new ArrayList<>(); queries.add(MatchQuery.of(m -> m @@ -216,33 +221,33 @@ protected ElasticSearchBase.SearchResult searchCollectionsByIds(List ids filters, null, null, - createSortOptions(sortBy), + createSortOptions(sortBy, CQLFields.class), null, null); } @Override - public ElasticSearchBase.SearchResult searchCollectionWithGeometry(List ids, String sortBy) { + public ElasticSearchBase.SearchResult searchCollectionWithGeometry(List ids, String sortBy) { return searchCollectionsByIds(ids, Boolean.TRUE, sortBy); } @Override - public ElasticSearchBase.SearchResult searchAllCollectionsWithGeometry(String sortBy) { + public ElasticSearchBase.SearchResult searchAllCollectionsWithGeometry(String sortBy) { return searchCollectionsByIds(null, Boolean.TRUE, sortBy); } @Override - public ElasticSearchBase.SearchResult searchCollections(List ids, String sortBy) { + public ElasticSearchBase.SearchResult searchCollections(List ids, String sortBy) { return searchCollectionsByIds(ids, Boolean.FALSE, sortBy); } @Override - public ElasticSearchBase.SearchResult searchAllCollections(String sortBy) { + public ElasticSearchBase.SearchResult searchAllCollections(String sortBy) { return searchCollectionsByIds(null, Boolean.FALSE, sortBy); } @Override - public ElasticSearchBase.SearchResult searchByParameters(List keywords, String cql, List properties, String sortBy, CQLCrsType coor) throws CQLException { + public ElasticSearchBase.SearchResult searchByParameters(List keywords, String cql, List properties, String sortBy, CQLCrsType coor) throws CQLException { if((keywords == null || keywords.isEmpty()) && cql == null) { return searchAllCollections(sortBy); @@ -334,50 +339,12 @@ public ElasticSearchBase.SearchResult searchByParameters(List keywords, filters, properties, searchAfter, - createSortOptions(sortBy), + createSortOptions(sortBy, CQLFields.class), score, maxSize ); } } - /** - * Parse and create a sort option - * ... - * - * @param sortBy - Must be of pattern + | -, + mean asc, - mean desc - * @return List of sort options - */ - protected List createSortOptions(String sortBy) { - if(sortBy == null || sortBy.isEmpty()) return null; - - String[] args = sortBy.split(","); - List sos = new ArrayList<>(); - - for(String arg: args) { - arg = arg.trim(); - if (arg.startsWith("-")) { - CQLFields field = Enum.valueOf(CQLFields.class, arg.substring(1).toLowerCase()); - - if(field.getSortBuilder() != null) { - ObjectBuilder sb = field.getSortBuilder().apply(SortOrder.Desc); - sos.add(sb.build()); - } - } - else { - // Default is +, there is a catch the + will be replaced as space in the property as + means space in url, by taking - // default is ASC we work around the problem, the trim removed the space - CQLFields field = arg.startsWith("+") ? - Enum.valueOf(CQLFields.class, arg.substring(1).toLowerCase()) : - Enum.valueOf(CQLFields.class, arg.toLowerCase()); - - if(field.getSortBuilder() != null) { - ObjectBuilder sb = field.getSortBuilder().apply(SortOrder.Asc); - sos.add(sb.build()); - } - } - } - return sos; - } @Override public BinaryResponse searchCollectionVectorTile(List ids, Integer tileMatrix, Integer tileRow, Integer tileCol) throws IOException { @@ -436,40 +403,104 @@ protected static FieldValue toFieldValue(String s) { // Assume it is string return FieldValue.of(s.trim()); } - + /** + * We will need to create a aggregation for each of the feature query, this one target the summary feature + * which create a summary of the indexed count group by geometry and date range for the cloud optimized data. + * @param collectionId - The metadata set id + * @param properties - The field you want to return + * @param filter - Any filter applied to the summary operation + * @return - Result + */ @Override - public DatasetSearchResult searchDatasetData( - String collectionId, - String startDate, - String endDate - ) { - List queries = new ArrayList<>(); - queries.add(MatchQuery.of(query -> query - .field("uuid") - .query(collectionId))._toQuery()); + public ElasticSearchBase.SearchResult searchFeatureSummary(String collectionId, List properties, String filter) { + + final String COORDINATES = "coordinates"; + final String TOTAL_COUNT = "total_count"; + final String MIN_TIME = "min_time"; + final String MAX_TIME = "max_time"; Supplier builderSupplier = () -> { SearchRequest.Builder builder = new SearchRequest.Builder(); - builder.index(datasetIndexName) - .size(this.getPageSize()) - .query(query -> query.bool(createBoolQueryForProperties(queries, null, null))); + + // Group by operation + TermsAggregation groupBy = TermsAggregation.of(term -> term + .script(script -> script.inline(s -> s + .lang("painless") + .source("doc['geometry.geometry.coordinates'].value.toString()")) + ) + .size(pageSize) + ); + // Sum of count + Aggregation sum = SumAggregation.of(s -> s.field(CQLFeatureFields.count.searchField))._toAggregation(); + + // Min value of field + Aggregation min = MinAggregation.of(s -> s.field(CQLFeatureFields.temporal.searchField))._toAggregation(); + + // Max value of field + Aggregation max = MaxAggregation.of(s -> s.field(CQLFeatureFields.temporal.searchField))._toAggregation(); + + // Field value of the group by + Aggregation field = new Aggregation.Builder().topHits(th -> th.size(1) + .source(src -> src.filter(f -> f + .includes("geometry.geometry.coordinates")) + ) + .sort(createSortOptions(CQLFeatureFields.id.name(), CQLFeatureFields.class))) + .build(); + + Aggregation aggregation = new Aggregation.Builder() + .terms(groupBy) + .aggregations(Map.of( + TOTAL_COUNT, sum, + MIN_TIME, min, + MAX_TIME, max, + COORDINATES, field + )) + .build(); + + builder.index(dataIndexName) + .size(0) // Do not return hits, only aggregations, that is the hits().hit() section will be empty + .aggregations(COORDINATES, aggregation); return builder; }; try { - Iterable> response = pagableSearch(builderSupplier, ObjectNode.class, (long) this.getPageSize()); + ElasticSearchBase.SearchResult result = new ElasticSearchBase.SearchResult<>(); + result.setCollections(new ArrayList<>()); + + Iterable response = pagableAggregation(builderSupplier, StringTermsBucket.class, null); + + for (StringTermsBucket node : response) { + if (node != null) { + StacItemModel. StacItemModelBuilder model = StacItemModel.builder(); - DatasetSearchResult result = new DatasetSearchResult(); + result.setTotal(result.getTotal() + node.docCount()); - for (var node : response) { - if (node != null && node.source() != null) { - var monthlyData = mapper.readValue(node.source().toPrettyString(), DatasetModel.class); - monthlyData.getData().forEach(result::addDatum); + TopHitsAggregate th = node.aggregations().get(COORDINATES).topHits(); + model.uuid(th.hits().hits().get(0).id()); + + JsonData jd = th.hits().hits().get(0).source(); + if(jd != null) { + Map map = jd.to(Map.class); + model.geometry((Map) map.get(FeatureProperty.GEOMETRY.getValue())); + } + + SumAggregate sa = node.aggregations().get(TOTAL_COUNT).sum(); + MinAggregate min = node.aggregations().get(MIN_TIME).min(); + MaxAggregate max = node.aggregations().get(MAX_TIME).max(); + + model.properties(Map.of( + FeatureProperty.COUNT.getValue(), sa.value(), + FeatureProperty.START_TIME.getValue(), min.valueAsString() == null ? "" : min.valueAsString(), + FeatureProperty.END_TIME.getValue(), max.valueAsString() == null ? "" : max.valueAsString() + )); + + result.getCollections().add(model.build()); } } return result; - } catch (Exception e) { + } + catch (Exception e) { log.error("Error while searching dataset.", e); } return null; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java index 571b3699..883b5b96 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java @@ -2,9 +2,11 @@ import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFields; +import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFieldsInterface; import au.org.aodn.ogcapi.server.core.model.enumeration.StacBasicField; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.*; +import co.elastic.clients.elasticsearch._types.aggregations.*; import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; import co.elastic.clients.elasticsearch._types.query_dsl.Query; import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders; @@ -14,6 +16,8 @@ import co.elastic.clients.elasticsearch.core.SearchResponse; import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.core.search.SourceConfig; +import co.elastic.clients.util.NamedValue; +import co.elastic.clients.util.ObjectBuilder; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -46,12 +50,49 @@ abstract class ElasticSearchBase { @Getter @Setter - public static class SearchResult { - Long total; + public static class SearchResult { + Long total = 0L; List sortValues; - List collections; + List collections; } + /** + * Parse and create a sort option + * ... + * + * @param sortBy - Must be of pattern + | -, + mean asc, - mean desc + * @return List of sort options + */ + protected & CQLFieldsInterface> List createSortOptions(String sortBy, Class enumClass) { + if(sortBy == null || sortBy.isEmpty()) return null; + String[] args = sortBy.split(","); + List sos = new ArrayList<>(); + + for(String arg: args) { + arg = arg.trim(); + if (arg.startsWith("-")) { + CQLFields field = Enum.valueOf(CQLFields.class, arg.substring(1).toLowerCase()); + + if(field.getSortBuilder() != null) { + ObjectBuilder sb = field.getSortBuilder().apply(SortOrder.Desc); + sos.add(sb.build()); + } + } + else { + // Default is +, there is a catch the + will be replaced as space in the property as + means space in url, by taking + // default is ASC we work around the problem, the trim removed the space + E field = arg.startsWith("+") ? + Enum.valueOf(enumClass, arg.substring(1).toLowerCase()) : + Enum.valueOf(enumClass, arg.toLowerCase()); + + if(field.getSortBuilder() != null) { + ObjectBuilder sb = field.getSortBuilder().apply(SortOrder.Asc); + sos.add(sb.build()); + } + } + } + return sos; + } /** * Construct the skeleton of in the elastic query and fill in values * @param must - The must portion of Elastic query @@ -107,7 +148,7 @@ protected SearchRequest buildSearchAsYouTypeRequest( * @param properties - The fields you want to return in the search, you can search a field but not include in the return * @return - The search result from Elastic query and format in StacCollectionModel */ - protected SearchResult searchCollectionBy(final List queries, + protected SearchResult searchCollectionBy(final List queries, final List should, final List filters, final List properties, @@ -191,7 +232,7 @@ protected SearchResult searchCollectionBy(final List queries, log.info("Start search {} {}", ZonedDateTime.now(), Thread.currentThread().getName()); Iterable> response = pagableSearch(builderSupplier, ObjectNode.class, maxSize); - SearchResult result = new SearchResult(); + SearchResult result = new SearchResult<>(); result.collections = new ArrayList<>(); result.total = countRecordsHit(builderSupplier); @@ -337,6 +378,97 @@ public Hit next() { } return Collections.emptySet(); } + /** + * There is a limit of how many record a query can return, this mean the record return may not be full set, you + * need to keep loading until you reach the end of records + * + * @param requestBuilder, assume it is sorted with order, what order isn't important, as long as it is sorted + * @param clazz - The type + * @return - The items that matches the query mentioned in the requestBuilder + * @param A generic type for Elastic query + */ + protected Iterable pagableAggregation( + Supplier requestBuilder, Class clazz, Long maxSize) { + try { + SearchRequest sr = requestBuilder.get().build(); + String aggKey = sr.aggregations().keySet().stream().findFirst().orElse(""); + log.debug("Final elastic aggregation payload {}", sr.toString()); + + final AtomicLong count = new AtomicLong(0); + final AtomicReference> response = new AtomicReference<>( + esClient.search(sr, clazz) + ); + + return () -> new Iterator<>() { + private int index = 0; + + @Override + public boolean hasNext() { + // No need continue if we already hit the end + if(maxSize != null) { + return count.get() < maxSize; + } + + Aggregate ags = response.get().aggregations().get(aggKey); + Buckets stb = ags.sterms().buckets(); + + // If we hit the end, that means we have iterated to end of page. + if (index < stb.array().size()) { + return true; + } + else { + // If last index is zero that mean nothing found already, so no need to look more + if (index == 0) return false; + + // Load next batch + try { + // Get the last sorted value from the last batch + List sortedValues = stb.array() + .get(index - 1) + .aggregations() + .get(aggKey) + .topHits() + .hits() + .hits() + .get(0) + .sort(); + + // Use the last builder and append the searchAfter values + SearchRequest request = requestBuilder.get().searchAfter(sortedValues).build(); + log.debug("Final elastic aggregation payload {}", request.toString()); + + response.set(esClient.search(request, clazz)); + // Reset counter from start + index = 0; + return index < response.get().hits().hits().size(); + } + catch(IOException ieo) { + throw new RuntimeException(ieo); + } + } + } + + @Override + public T next() { + count.incrementAndGet(); + + Aggregate ags = response.get().aggregations().get(aggKey); + Buckets stb = ags.sterms().buckets(); + + if(index < stb.array().size()) { + return clazz.cast(stb.array().get(index++)); + } + else { + return null; + } + } + }; + } + catch(IOException e) { + log.error("Fail to fetch record", e); + } + return Collections.emptySet(); + } protected StacCollectionModel formatResult(ObjectNode nodes) { try { diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java index 596d0305..36c75d4a 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java @@ -94,7 +94,7 @@ public ResponseEntity getCollectionList(List keywords, * @param filter - Any existing filter * @return - A combined filter with datetime rewrite. */ - public static String processDatetimeParameter(String datetime, String filter) { + public static String processDatetimeParameter(String fieldName, String datetime, String filter) { // TODO: How to handle this? e.g how to know if it is before or after if ?datetime= @@ -118,7 +118,7 @@ else if (datetime.contains("/") && !datetime.contains("..")) { } if(d != null) { - f = String.format("temporal %s %s", operator, d); + f = String.format("%s %s %s", fieldName, operator, d); } if((filter == null || filter.isEmpty())) { @@ -139,15 +139,22 @@ else if (datetime.contains("/") && !datetime.contains("..")) { * @param filter * @return */ - public static String processBBoxParameter(List bbox, String filter) { + public static String processBBoxParameter(String fieldName, List bbox, String filter) { String f = null; if(bbox.size() == 4) { // 2D - f = String.format("BBOX(%s,%s,%s,%s)", bbox.get(0), bbox.get(1), bbox.get(2), bbox.get(3)); + f = String.format("BBOX(%s,%s,%s,%s,%s)", fieldName, bbox.get(0), bbox.get(1), bbox.get(2), bbox.get(3)); } else if(bbox.size() == 6) { // 3D - f = String.format("BBOX(%s,%s,%%,%s,%s,%s)", bbox.get(0), bbox.get(1), bbox.get(2), bbox.get(3), bbox.get(4), bbox.get(5)); + f = String.format("BBOX(%s,%s,%s,%s,%s,%s,%s)", fieldName, bbox.get(0), bbox.get(1), bbox.get(2), bbox.get(3), bbox.get(4), bbox.get(5)); + } + + if(f == null) { + return filter; + } + else { + return String.join(" AND ", filter, f); } } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/Search.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/Search.java index 8e759324..256b6ca0 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/Search.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/Search.java @@ -1,6 +1,7 @@ package au.org.aodn.ogcapi.server.core.service; -import au.org.aodn.ogcapi.server.core.model.DatasetSearchResult; +import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; +import au.org.aodn.ogcapi.server.core.model.StacItemModel; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType; import co.elastic.clients.transport.endpoints.BinaryResponse; import org.springframework.http.ResponseEntity; @@ -10,14 +11,14 @@ import java.util.Map; public interface Search { - ElasticSearchBase.SearchResult searchCollectionWithGeometry(List ids, String sortBy) throws Exception; - ElasticSearchBase.SearchResult searchAllCollectionsWithGeometry(String sortBy) throws Exception; + ElasticSearchBase.SearchResult searchCollectionWithGeometry(List ids, String sortBy) throws Exception; + ElasticSearchBase.SearchResult searchAllCollectionsWithGeometry(String sortBy) throws Exception; - ElasticSearchBase.SearchResult searchCollections(List ids, String sortBy); - ElasticSearchBase.SearchResult searchAllCollections(String sortBy) throws Exception; - DatasetSearchResult searchDatasetData(String collectionId, String startDate, String endDate) throws Exception; + ElasticSearchBase.SearchResult searchCollections(List ids, String sortBy); + ElasticSearchBase.SearchResult searchAllCollections(String sortBy) throws Exception; + ElasticSearchBase.SearchResult searchFeatureSummary(String collectionId, List properties, String filter) throws Exception; - ElasticSearchBase.SearchResult searchByParameters( + ElasticSearchBase.SearchResult searchByParameters( List targets, String filter, List properties, diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/CommonUtils.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/CommonUtils.java new file mode 100644 index 00000000..1e7a1e45 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/CommonUtils.java @@ -0,0 +1,17 @@ +package au.org.aodn.ogcapi.server.core.util; + +import java.util.Optional; +import java.util.function.Supplier; + +public class CommonUtils { + public static Optional safeGet(Supplier supplier) { + try { + return Optional.of(supplier.get()); + } catch ( + NullPointerException + | IndexOutOfBoundsException + | ClassCastException ignored) { + return Optional.empty(); + } + } +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java index 445e267b..05bb2613 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java @@ -3,10 +3,21 @@ import au.org.aodn.ogcapi.features.api.CollectionsApi; import au.org.aodn.ogcapi.features.model.Collection; import au.org.aodn.ogcapi.features.model.Collections; +import au.org.aodn.ogcapi.features.model.Exception; import au.org.aodn.ogcapi.features.model.FeatureCollectionGeoJSON; import au.org.aodn.ogcapi.features.model.FeatureGeoJSON; +import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFields; +import au.org.aodn.ogcapi.server.core.model.enumeration.FeatureId; import au.org.aodn.ogcapi.server.core.service.OGCApiService; import io.swagger.v3.oas.annotations.Hidden; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import jakarta.validation.Valid; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -27,27 +38,74 @@ public ResponseEntity describeCollection(String collectionId) { return featuresService.getCollection(collectionId, null); } + @Hidden @Override public ResponseEntity getFeature(String collectionId, String featureId) { return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build(); } -// @RequestMapping( -// value = {"/collections/{collectionId}/items"}, -// produces = {"application/geo+json", "text/html", "application/json"}, -// method = {RequestMethod.GET} -// ) -// public ResponseEntity getFeatures( -// -// @PathVariable("collectionId") String collectionId, -// @RequestParam(value = "start_datetime", required = false) String startDate, -// @RequestParam(value = "end_datetime", required = false) String endDate, -// // keep these two parameters for future usage -// @RequestParam(value= "zoom", required = false) Double zoomLevel, -// @RequestParam(value="bbox", required = false) List bbox -// ) { -// return featuresService.getSummarizedDataset(collectionId, startDate, endDate); -// } + @Operation( + summary = "fetch a single feature", + description = "Fetch the feature with id `featureId` in the feature collection with id `collectionId`. Use content negotiation to request HTML or GeoJSON.", + tags = {"Data"} + ) + @ApiResponses({@ApiResponse( + responseCode = "200", + description = "fetch the feature with id `featureId` in the feature collection with id `collectionId`", + content = {@Content( + mediaType = "application/geo+json", + schema = @Schema( + implementation = FeatureGeoJSON.class + ) + )} + ), @ApiResponse( + responseCode = "404", + description = "The requested resource does not exist on the server. For example, a path parameter had an incorrect value." + ), @ApiResponse( + responseCode = "500", + description = "A server error occurred.", + content = {@Content( + mediaType = "application/json", + schema = @Schema( + implementation = Exception.class + ) + )} + )}) + @RequestMapping( + value = {"/collections/{collectionId}/items/{featureId}"}, + produces = {"application/geo+json", "text/html", "application/json"}, + method = {RequestMethod.GET} + ) + ResponseEntity getFeature( + @Parameter(in = ParameterIn.PATH,description = "local identifier of a collection",required = true,schema = @Schema) + @PathVariable("collectionId") String collectionId, + @Parameter(in = ParameterIn.PATH,description = "local identifier of a feature",required = true,schema = @Schema) + @PathVariable("featureId") String featureId, + @Parameter(in = ParameterIn.QUERY, description = "Property to be return" ,schema=@Schema()) + @Valid @RequestParam(value = "properties", required = false) List properties, + @Parameter(in = ParameterIn.QUERY, description = "Only records that have a geometry that intersects the bounding box are selected. The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or depth): * Lower left corner, coordinate axis 1 * Lower left corner, coordinate axis 2 * Minimum value, coordinate axis 3 (optional) * Upper right corner, coordinate axis 1 * Upper right corner, coordinate axis 2 * Maximum value, coordinate axis 3 (optional) The coordinate reference system of the values is WGS 84 long/lat (http://www.opengis.net/def/crs/OGC/1.3/CRS84) unless a different coordinate reference system is specified in the parameter `bbox-crs`. For WGS 84 longitude/latitude the values are in most cases the sequence of minimum longitude, minimum latitude, maximum longitude and maximum latitude. However, in cases where the box spans the antimeridian the first value (west-most box edge) is larger than the third value (east-most box edge). If the vertical axis is included, the third and the sixth number are the bottom and the top of the 3-dimensional bounding box. If a record has multiple spatial geometry properties, it is the decision of the server whether only a single spatial geometry property is used to determine the extent or all relevant geometries." ,schema=@Schema()) + @Valid @RequestParam(value = "bbox", required = false) List bbox, + @Parameter(in = ParameterIn.QUERY, description = "Either a date-time or an interval, open or closed. Date and time expressions adhere to RFC 3339. Open intervals are expressed using double-dots. Examples: * A date-time: \"2018-02-12T23:20:50Z\" * A closed interval: \"2018-02-12T00:00:00Z/2018-03-18T12:31:12Z\" * Open intervals: \"2018-02-12T00:00:00Z/..\" or \"../2018-03-18T12:31:12Z\" Only records that have a temporal property that intersects the value of `datetime` are selected. It is left to the decision of the server whether only a single temporal property is used to determine the extent or all relevant temporal properties." ,schema=@Schema()) + @Valid @RequestParam(value = "datetime", required = false) String datetime) { + + String filter = null; + if (datetime != null) { + filter = OGCApiService.processDatetimeParameter(CQLFields.temporal.name(), datetime, filter); + } + + if (bbox != null) { + filter = OGCApiService.processBBoxParameter(CQLFields.geometry.name(), bbox, filter); + } + + try { + FeatureId fid = FeatureId.valueOf(FeatureId.class, featureId); + return featuresService.getFeature(collectionId, fid, properties, filter != null ? "filter=" + filter : null); + } + catch(java.lang.Exception e) { + return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build(); + } + } + /** * * @param collectionId - The collection id @@ -58,11 +116,7 @@ public ResponseEntity getFeature(String collectionId, String fea */ @Override public ResponseEntity getFeatures(String collectionId, Integer limit, List bbox, String datetime) { - String filter = null; - if (datetime != null) { - filter = OGCApiService.processDatetimeParameter(datetime, null); - } - return featuresService.getSummarizedDataset(collectionId, startDate, endDate); + return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build(); } /** * Hidden effectively disable this REST point because it is common in many places and diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java index fbc49257..d4391d14 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java @@ -3,10 +3,15 @@ import au.org.aodn.ogcapi.features.model.Collection; import au.org.aodn.ogcapi.features.model.FeatureCollectionGeoJSON; import au.org.aodn.ogcapi.server.core.mapper.StacToCollection; +import au.org.aodn.ogcapi.server.core.model.DataSearchResult; +import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; +import au.org.aodn.ogcapi.server.core.model.StacItemModel; +import au.org.aodn.ogcapi.server.core.model.enumeration.FeatureId; import au.org.aodn.ogcapi.server.core.service.ElasticSearch; import au.org.aodn.ogcapi.server.core.service.OGCApiService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; @@ -26,7 +31,7 @@ public List getConformanceDeclaration() { } public ResponseEntity getCollection(String id, String sortBy) throws NoSuchElementException { - ElasticSearch.SearchResult model = search.searchCollections(List.of(id), sortBy); + ElasticSearch.SearchResult model = search.searchCollections(List.of(id), sortBy); if (!model.getCollections().isEmpty()) { if(model.getCollections().size() > 1) { @@ -41,18 +46,22 @@ public ResponseEntity getCollection(String id, String sortBy) throws } } - public ResponseEntity getSummarizedDataset( - String collectionId, - String startDate, - String endDate - ) { - try { - var result = search.searchDatasetData(collectionId, startDate, endDate); - return ResponseEntity.ok() - .body(result.getSummarizedDataset()); - } catch (Exception e) { - log.error("Error while getting dataset", e); - return ResponseEntity.internalServerError().build(); + public ResponseEntity getFeature(String collectionId, + FeatureId fid, + List properties, + String filter) throws Exception { + switch(fid) { + case summary -> { + ElasticSearch.SearchResult result = search.searchFeatureSummary(collectionId, properties, filter); + return ResponseEntity.ok() + .body(result.getCollections()); + } + default -> { + // Individual item + return new ResponseEntity<>(HttpStatus.NOT_IMPLEMENTED); + } } } + + } diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml index 96305610..7c866e21 100644 --- a/server/src/main/resources/application.yaml +++ b/server/src/main/resources/application.yaml @@ -10,8 +10,8 @@ elasticsearch: name: dev_portal_records vocabs_index: name: vocabs_index - dataset_index: - name: dataset_index + cloud_optimized_index: + name: es-coindexer-dev serverUrl: http://localhost:9200 apiKey: search_as_you_type: diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/OGCApiServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/OGCApiServiceTest.java index d36c68c3..162e6584 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/OGCApiServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/OGCApiServiceTest.java @@ -1,5 +1,6 @@ package au.org.aodn.ogcapi.server.core.service; +import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFields; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -12,22 +13,22 @@ public class OGCApiServiceTest { @Test public void verifyProcessDatetimeParameter() { - String o = OGCApiService.processDatetimeParameter("../2021-10-10", ""); + String o = OGCApiService.processDatetimeParameter(CQLFields.temporal.name(), "../2021-10-10", ""); assertEquals( "temporal before 2021-10-10", o, "Before incorrect1"); - o = OGCApiService.processDatetimeParameter("/2021-10-10", ""); + o = OGCApiService.processDatetimeParameter(CQLFields.temporal.name(), "/2021-10-10", ""); assertEquals( "temporal before 2021-10-10", o, "Before incorrect2"); - o = OGCApiService.processDatetimeParameter("2021-10-10/", ""); + o = OGCApiService.processDatetimeParameter(CQLFields.temporal.name(),"2021-10-10/", ""); assertEquals( "temporal after 2021-10-10", o, "After incorrect1"); - o = OGCApiService.processDatetimeParameter("2021-10-10/..", ""); + o = OGCApiService.processDatetimeParameter(CQLFields.temporal.name(),"2021-10-10/..", ""); assertEquals( "temporal after 2021-10-10", o, "After incorrect1"); - o = OGCApiService.processDatetimeParameter("2021-10-10/2022-10-10", ""); + o = OGCApiService.processDatetimeParameter(CQLFields.temporal.name(),"2021-10-10/2022-10-10", ""); assertEquals( "temporal during 2021-10-10/2022-10-10", o, "During incorrect1"); - o = OGCApiService.processDatetimeParameter("/2021-10-10", "geometry is null"); + o = OGCApiService.processDatetimeParameter(CQLFields.temporal.name(),"/2021-10-10", "geometry is null"); assertEquals( "geometry is null AND temporal before 2021-10-10", o, "Before plus filter incorrect1"); } } From 76ad4038803ed872e3dc90bbb2a4860e12676d88 Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 15 Jan 2025 15:33:07 +1100 Subject: [PATCH 03/10] First work group by --- .../model/enumeration/CQLFeatureFields.java | 6 +-- .../server/core/service/ElasticSearch.java | 44 ++++++++++++------- .../core/service/ElasticSearchBase.java | 30 +++++-------- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java index a547b3ad..7fff0c5c 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java @@ -17,10 +17,10 @@ public enum CQLFeatureFields implements CQLFieldsInterface { id( - "id", - "id", + StacBasicField.UUID.searchField, + StacBasicField.UUID.displayField, null, - (order) -> new SortOptions.Builder().field(f -> f.field("id.keyword").order(order)) + (order) -> new SortOptions.Builder().field(f -> f.field(StacBasicField.UUID.sortField).order(order)) ), collection( "collection", diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 83221b24..8829b9b2 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -1,6 +1,5 @@ package au.org.aodn.ogcapi.server.core.service; -import au.org.aodn.ogcapi.server.core.model.DataSearchResult; import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; import au.org.aodn.ogcapi.server.core.model.StacItemModel; import au.org.aodn.ogcapi.server.core.model.dto.SearchSuggestionsDto; @@ -9,8 +8,6 @@ import au.org.aodn.ogcapi.server.core.parser.elastic.QueryHandler; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.FieldValue; -import co.elastic.clients.elasticsearch._types.SortOptions; -import co.elastic.clients.elasticsearch._types.SortOrder; import co.elastic.clients.elasticsearch._types.aggregations.*; import co.elastic.clients.elasticsearch._types.query_dsl.*; import co.elastic.clients.elasticsearch.core.SearchMvtRequest; @@ -20,11 +17,7 @@ import co.elastic.clients.elasticsearch.core.search_mvt.GridType; import co.elastic.clients.json.JsonData; import co.elastic.clients.transport.endpoints.BinaryResponse; -import co.elastic.clients.util.ObjectBuilder; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import jakarta.json.JsonValue; import lombok.extern.slf4j.Slf4j; import org.geotools.filter.text.commons.CompilerUtil; import org.geotools.filter.text.commons.Language; @@ -36,7 +29,7 @@ import java.io.IOException; import java.util.*; -import java.util.function.Supplier; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -406,6 +399,9 @@ protected static FieldValue toFieldValue(String s) { /** * We will need to create a aggregation for each of the feature query, this one target the summary feature * which create a summary of the indexed count group by geometry and date range for the cloud optimized data. + * Example: + * {"aggregations":{"coordinates":{"aggregations":{"min_time":{"min":{"field":"properties.time"}},"coordinates":{"top_hits":{"size":1,"sort":[{"id.keyword":{"order":"asc"}}],"_source":{"includes":["geometry.geometry.coordinates"]}}},"total_count":{"sum":{"field":"properties.count"}},"max_time":{"max":{"field":"properties.time"}}},"composite":{"size":2200,"sources":[{"coordinates":{"terms":{"script":{"lang":"painless","source":"doc['geometry.geometry.coordinates'].value.toString()"}}}},{"id":{"terms":{"field":"id.keyword"}}}]}}},"size":0} + * * @param collectionId - The metadata set id * @param properties - The field you want to return * @param filter - Any filter applied to the summary operation @@ -419,17 +415,31 @@ public ElasticSearchBase.SearchResult searchFeatureSummary(String final String MIN_TIME = "min_time"; final String MAX_TIME = "max_time"; - Supplier builderSupplier = () -> { + Function, SearchRequest.Builder> builderSupplier = (afterKey) -> { SearchRequest.Builder builder = new SearchRequest.Builder(); - // Group by operation - TermsAggregation groupBy = TermsAggregation.of(term -> term + CompositeAggregationSource coordinate = CompositeAggregationSource.of(c -> c.terms(t -> t .script(script -> script.inline(s -> s .lang("painless") .source("doc['geometry.geometry.coordinates'].value.toString()")) - ) - .size(pageSize) - ); + ))); + + CompositeAggregationSource id = CompositeAggregationSource.of(c -> c.terms(t -> t + .field(StacBasicField.UUID.sortField))); + + Aggregation groupBy = afterKey == null ? + new Aggregation.Builder().composite(c -> c + .sources(List.of(Map.of(COORDINATES, coordinate), Map.of("id", id))) + .size(pageSize) + ).build() + : + new Aggregation.Builder().composite(c -> c + .sources(List.of(Map.of(COORDINATES, coordinate), Map.of("id", id))) + .size(pageSize) + .after(afterKey) + ).build(); + + // Sum of count Aggregation sum = SumAggregation.of(s -> s.field(CQLFeatureFields.count.searchField))._toAggregation(); @@ -448,7 +458,7 @@ public ElasticSearchBase.SearchResult searchFeatureSummary(String .build(); Aggregation aggregation = new Aggregation.Builder() - .terms(groupBy) + .composite(groupBy.composite()) .aggregations(Map.of( TOTAL_COUNT, sum, MIN_TIME, min, @@ -468,9 +478,9 @@ public ElasticSearchBase.SearchResult searchFeatureSummary(String ElasticSearchBase.SearchResult result = new ElasticSearchBase.SearchResult<>(); result.setCollections(new ArrayList<>()); - Iterable response = pagableAggregation(builderSupplier, StringTermsBucket.class, null); + Iterable response = pageableAggregation(builderSupplier, CompositeBucket.class, null); - for (StringTermsBucket node : response) { + for (CompositeBucket node : response) { if (node != null) { StacItemModel. StacItemModelBuilder model = StacItemModel.builder(); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java index 883b5b96..f98ceabb 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java @@ -16,7 +16,6 @@ import co.elastic.clients.elasticsearch.core.SearchResponse; import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.core.search.SourceConfig; -import co.elastic.clients.util.NamedValue; import co.elastic.clients.util.ObjectBuilder; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -30,6 +29,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -230,7 +230,7 @@ protected SearchResult searchCollectionBy(final List try { log.info("Start search {} {}", ZonedDateTime.now(), Thread.currentThread().getName()); - Iterable> response = pagableSearch(builderSupplier, ObjectNode.class, maxSize); + Iterable> response = pageableSearch(builderSupplier, ObjectNode.class, maxSize); SearchResult result = new SearchResult<>(); result.collections = new ArrayList<>(); @@ -313,7 +313,7 @@ protected Long countRecordsHit(Supplier requestBuilder) { * @return - The items that matches the query mentioned in the requestBuilder * @param A generic type for Elastic query */ - protected Iterable> pagableSearch(Supplier requestBuilder, Class clazz, Long maxSize) { + protected Iterable> pageableSearch(Supplier requestBuilder, Class clazz, Long maxSize) { try { SearchRequest sr = requestBuilder.get().build(); log.debug("Final elastic search payload {}", sr.toString()); @@ -387,12 +387,12 @@ public Hit next() { * @return - The items that matches the query mentioned in the requestBuilder * @param A generic type for Elastic query */ - protected Iterable pagableAggregation( - Supplier requestBuilder, Class clazz, Long maxSize) { + protected Iterable pageableAggregation( + Function, SearchRequest.Builder> requestBuilder, Class clazz, Long maxSize) { try { - SearchRequest sr = requestBuilder.get().build(); + SearchRequest sr = requestBuilder.apply(null).build(); String aggKey = sr.aggregations().keySet().stream().findFirst().orElse(""); - log.debug("Final elastic aggregation payload {}", sr.toString()); + log.debug("Final elastic aggregation payload {}", sr); final AtomicLong count = new AtomicLong(0); final AtomicReference> response = new AtomicReference<>( @@ -410,7 +410,7 @@ public boolean hasNext() { } Aggregate ags = response.get().aggregations().get(aggKey); - Buckets stb = ags.sterms().buckets(); + Buckets stb = ags.composite().buckets(); // If we hit the end, that means we have iterated to end of page. if (index < stb.array().size()) { @@ -423,18 +423,8 @@ public boolean hasNext() { // Load next batch try { // Get the last sorted value from the last batch - List sortedValues = stb.array() - .get(index - 1) - .aggregations() - .get(aggKey) - .topHits() - .hits() - .hits() - .get(0) - .sort(); - // Use the last builder and append the searchAfter values - SearchRequest request = requestBuilder.get().searchAfter(sortedValues).build(); + SearchRequest request = requestBuilder.apply(ags.composite().afterKey()).build(); log.debug("Final elastic aggregation payload {}", request.toString()); response.set(esClient.search(request, clazz)); @@ -453,7 +443,7 @@ public T next() { count.incrementAndGet(); Aggregate ags = response.get().aggregations().get(aggKey); - Buckets stb = ags.sterms().buckets(); + Buckets stb = ags.composite().buckets(); if(index < stb.array().size()) { return clazz.cast(stb.array().get(index++)); From 80eee67e3353ae48d7ede60a09c1ded4fe548d29 Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 15 Jan 2025 15:38:02 +1100 Subject: [PATCH 04/10] Remove toString() --- .../org/aodn/ogcapi/server/core/service/ElasticSearchBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java index f98ceabb..a52e9432 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java @@ -316,7 +316,7 @@ protected Long countRecordsHit(Supplier requestBuilder) { protected Iterable> pageableSearch(Supplier requestBuilder, Class clazz, Long maxSize) { try { SearchRequest sr = requestBuilder.get().build(); - log.debug("Final elastic search payload {}", sr.toString()); + log.debug("Final elastic search payload {}", sr); final AtomicLong count = new AtomicLong(0); final AtomicReference> response = new AtomicReference<>( From 6147a3eeb798e2664f5f5d473d20d4ac6246adda Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 15 Jan 2025 17:18:38 +1100 Subject: [PATCH 05/10] Api work now --- .../core/mapper/StacToFeatureCollection.java | 49 +++++++++++++++++++ .../server/core/service/OGCApiService.java | 28 +++++++++-- .../aodn/ogcapi/server/features/RestApi.java | 12 ++++- .../ogcapi/server/features/RestServices.java | 26 +--------- 4 files changed, 86 insertions(+), 29 deletions(-) create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToFeatureCollection.java diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToFeatureCollection.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToFeatureCollection.java new file mode 100644 index 00000000..63f096d2 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToFeatureCollection.java @@ -0,0 +1,49 @@ +package au.org.aodn.ogcapi.server.core.mapper; + +import au.org.aodn.ogcapi.features.model.FeatureCollectionGeoJSON; +import au.org.aodn.ogcapi.features.model.FeatureGeoJSON; +import au.org.aodn.ogcapi.features.model.PointGeoJSON; +import au.org.aodn.ogcapi.server.core.model.StacItemModel; +import au.org.aodn.ogcapi.server.core.service.ElasticSearch; +import com.fasterxml.jackson.core.type.TypeReference; +import org.mapstruct.Mapper; +import org.opengis.filter.Filter; +import org.springframework.stereotype.Service; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; + +import static au.org.aodn.ogcapi.server.core.util.CommonUtils.safeGet; + +@Service +@Mapper(componentModel = "spring") +public abstract class StacToFeatureCollection implements Converter, FeatureCollectionGeoJSON> { + + @Override + public FeatureCollectionGeoJSON convert(ElasticSearch.SearchResult model, Filter filter) { + FeatureCollectionGeoJSON f = new FeatureCollectionGeoJSON(); + f.setType(FeatureCollectionGeoJSON.TypeEnum.FEATURECOLLECTION); + + List features = model.getCollections().stream() + .map(i -> { + FeatureGeoJSON feature = new FeatureGeoJSON(); + feature.setType(FeatureGeoJSON.TypeEnum.FEATURE); + + safeGet(() -> ((Map>)i.getGeometry().get("geometry")).get("coordinates")) + .ifPresent(g -> + feature.setGeometry(new PointGeoJSON() + .type(PointGeoJSON.TypeEnum.POINT) + .coordinates(g) + )); + + feature.setProperties(i.getProperties()); + + return feature; + }) + .toList(); + + f.setFeatures(features); + return f; + } +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java index 36c75d4a..3841423a 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java @@ -1,6 +1,9 @@ package au.org.aodn.ogcapi.server.core.service; +import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; +import au.org.aodn.ogcapi.server.core.model.StacItemModel; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType; +import au.org.aodn.ogcapi.server.core.model.enumeration.FeatureId; import au.org.aodn.ogcapi.server.core.model.enumeration.OGCMediaTypeMapper; import au.org.aodn.ogcapi.server.core.exception.CustomException; import au.org.aodn.ogcapi.server.core.parser.stac.CQLToStacFilterFactory; @@ -29,22 +32,41 @@ public abstract class OGCApiService { protected Search search; /** - * You can find conformance id here https://docs.ogc.org/is/19-072/19-072.html#ats_core + * You can find conformance id + * here * @return List of string contains conformance */ public abstract List getConformanceDeclaration(); + public ResponseEntity getFeature(String collectionId, + FeatureId fid, + List properties, + String filter, + BiFunction, Filter, R> converter) throws Exception { + switch(fid) { + case summary -> { + ElasticSearch.SearchResult result = search.searchFeatureSummary(collectionId, properties, filter); + return ResponseEntity.ok() + .body(converter.apply(result, null)); + } + default -> { + // Individual item + return new ResponseEntity<>(HttpStatus.NOT_IMPLEMENTED); + } + } + } + public ResponseEntity getCollectionList(List keywords, String filter, List properties, String sortBy, OGCMediaTypeMapper f, CQLCrsType coor, - BiFunction converter) { + BiFunction, Filter, R> converter) { try { switch (f) { case json -> { - ElasticSearchBase.SearchResult result = search.searchByParameters(keywords, filter, properties, sortBy, coor); + ElasticSearchBase.SearchResult result = search.searchByParameters(keywords, filter, properties, sortBy, coor); CQLToStacFilterFactory factory = CQLToStacFilterFactory.builder() .cqlCrsType(coor) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java index 05bb2613..f0c26118 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java @@ -6,6 +6,7 @@ import au.org.aodn.ogcapi.features.model.Exception; import au.org.aodn.ogcapi.features.model.FeatureCollectionGeoJSON; import au.org.aodn.ogcapi.features.model.FeatureGeoJSON; +import au.org.aodn.ogcapi.server.core.mapper.StacToFeatureCollection; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFields; import au.org.aodn.ogcapi.server.core.model.enumeration.FeatureId; import au.org.aodn.ogcapi.server.core.service.OGCApiService; @@ -33,6 +34,9 @@ public class RestApi implements CollectionsApi { @Autowired protected RestServices featuresService; + @Autowired + protected StacToFeatureCollection stacToFeatureCollection; + @Override public ResponseEntity describeCollection(String collectionId) { return featuresService.getCollection(collectionId, null); @@ -99,7 +103,13 @@ ResponseEntity getFeature( try { FeatureId fid = FeatureId.valueOf(FeatureId.class, featureId); - return featuresService.getFeature(collectionId, fid, properties, filter != null ? "filter=" + filter : null); + return featuresService.getFeature( + collectionId, + fid, + properties, + filter != null ? "filter=" + filter : null, + stacToFeatureCollection::convert + ); } catch(java.lang.Exception e) { return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build(); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java index d4391d14..f0e68ab1 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java @@ -1,25 +1,20 @@ package au.org.aodn.ogcapi.server.features; import au.org.aodn.ogcapi.features.model.Collection; -import au.org.aodn.ogcapi.features.model.FeatureCollectionGeoJSON; import au.org.aodn.ogcapi.server.core.mapper.StacToCollection; -import au.org.aodn.ogcapi.server.core.model.DataSearchResult; import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; -import au.org.aodn.ogcapi.server.core.model.StacItemModel; -import au.org.aodn.ogcapi.server.core.model.enumeration.FeatureId; import au.org.aodn.ogcapi.server.core.service.ElasticSearch; import au.org.aodn.ogcapi.server.core.service.OGCApiService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import java.util.List; import java.util.NoSuchElementException; -@Service("FeaturesRestService") @Slf4j +@Service("FeaturesRestService") public class RestServices extends OGCApiService { @Autowired @@ -45,23 +40,4 @@ public ResponseEntity getCollection(String id, String sortBy) throws return ResponseEntity.notFound().build(); } } - - public ResponseEntity getFeature(String collectionId, - FeatureId fid, - List properties, - String filter) throws Exception { - switch(fid) { - case summary -> { - ElasticSearch.SearchResult result = search.searchFeatureSummary(collectionId, properties, filter); - return ResponseEntity.ok() - .body(result.getCollections()); - } - default -> { - // Individual item - return new ResponseEntity<>(HttpStatus.NOT_IMPLEMENTED); - } - } - } - - } From da99942a18260524e439c9644e0ed2579c36861a Mon Sep 17 00:00:00 2001 From: rng Date: Wed, 15 Jan 2025 17:27:27 +1100 Subject: [PATCH 06/10] Avoid null geometry --- .../aodn/ogcapi/server/core/mapper/StacToFeatureCollection.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToFeatureCollection.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToFeatureCollection.java index 63f096d2..c11c7c62 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToFeatureCollection.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToFeatureCollection.java @@ -41,6 +41,7 @@ public FeatureCollectionGeoJSON convert(ElasticSearch.SearchResult i.getGeometry() != null) .toList(); f.setFeatures(features); From c3547005351f1210b9acf505500d99cf49b549bb Mon Sep 17 00:00:00 2001 From: rng Date: Thu, 16 Jan 2025 10:13:59 +1100 Subject: [PATCH 07/10] Fix and simplify query as no time due to its aggregate nature --- .../model/enumeration/CQLFeatureFields.java | 8 +- .../model/enumeration/StacBasicField.java | 3 +- .../server/core/service/ElasticSearch.java | 94 +++++++++++++++---- 3 files changed, 84 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java index 7fff0c5c..703ed73c 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java @@ -23,10 +23,10 @@ public enum CQLFeatureFields implements CQLFieldsInterface { (order) -> new SortOptions.Builder().field(f -> f.field(StacBasicField.UUID.sortField).order(order)) ), collection( - "collection", - "collection", + StacBasicField.Collection.searchField, + StacBasicField.Collection.displayField, null, - null + (order) -> new SortOptions.Builder().field(f -> f.field(StacBasicField.Collection.sortField).order(order)) ), temporal( "properties.time", @@ -44,7 +44,7 @@ public enum CQLFeatureFields implements CQLFieldsInterface { "geometry", "geometry", null, - null + (order) -> new SortOptions.Builder().field(f -> f.field("geometry.geometry.coordinates").order(order)) ); // Field that use to do sort, elastic search treat FieldData (searchField) differently, a searchField is not diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/StacBasicField.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/StacBasicField.java index 15072335..f6df6dd2 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/StacBasicField.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/StacBasicField.java @@ -20,7 +20,8 @@ public enum StacBasicField { "organisation_vocabs", "summaries.organisation_vocabs" ), - Links("links", "links") + Links("links", "links"), + Collection("collection", "collection", "collection.keyword"), ; // Field that use to do sort, elastic search treat FieldData (searchField) differently, a searchField is not diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 8829b9b2..95fd1b2f 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -399,8 +399,67 @@ protected static FieldValue toFieldValue(String s) { /** * We will need to create a aggregation for each of the feature query, this one target the summary feature * which create a summary of the indexed count group by geometry and date range for the cloud optimized data. - * Example: - * {"aggregations":{"coordinates":{"aggregations":{"min_time":{"min":{"field":"properties.time"}},"coordinates":{"top_hits":{"size":1,"sort":[{"id.keyword":{"order":"asc"}}],"_source":{"includes":["geometry.geometry.coordinates"]}}},"total_count":{"sum":{"field":"properties.count"}},"max_time":{"max":{"field":"properties.time"}}},"composite":{"size":2200,"sources":[{"coordinates":{"terms":{"script":{"lang":"painless","source":"doc['geometry.geometry.coordinates'].value.toString()"}}}},{"id":{"terms":{"field":"id.keyword"}}}]}}},"size":0} + * Below code equals this: + * { + * "aggregations": { + * "coordinates": { + * "aggregations": { + * "total_count": { + * "sum": { + * "field": "properties.count" + * } + * }, + * "max_time": { + * "max": { + * "field": "properties.time" + * } + * }, + * "min_time": { + * "min": { + * "field": "properties.time" + * } + * }, + * "coordinates": { + * "top_hits": { + * "size": 1, + * "sort": [ + * { + * "collection.keyword": { + * "order": "asc" + * } + * }, + * { + * "geometry.geometry.coordinates": { + * "order": "asc" + * } + * }, + * ] + * } + * } + * }, + * "composite": { + * "size": 2200, + * "sources": [ + * { + * "collection": { + * "terms": { + * "field": "collection.keyword" + * } + * } + * }, + * { + * "coordinates": { + * "terms": { + * "field": "geometry.geometry.coordinates" + * } + * } + * } + * ] + * } + * } + * }, + * "size": 0 + * } * * @param collectionId - The metadata set id * @param properties - The field you want to return @@ -417,24 +476,28 @@ public ElasticSearchBase.SearchResult searchFeatureSummary(String Function, SearchRequest.Builder> builderSupplier = (afterKey) -> { SearchRequest.Builder builder = new SearchRequest.Builder(); - + // Group by coordinates CompositeAggregationSource coordinate = CompositeAggregationSource.of(c -> c.terms(t -> t - .script(script -> script.inline(s -> s - .lang("painless") - .source("doc['geometry.geometry.coordinates'].value.toString()")) - ))); - + .field("geometry.geometry.coordinates"))); + // Group by id CompositeAggregationSource id = CompositeAggregationSource.of(c -> c.terms(t -> t - .field(StacBasicField.UUID.sortField))); + .field(StacBasicField.Collection.sortField))); + // Use t page to another batch of records if exist Aggregation groupBy = afterKey == null ? new Aggregation.Builder().composite(c -> c - .sources(List.of(Map.of(COORDINATES, coordinate), Map.of("id", id))) + .sources(List.of( + Map.of(StacBasicField.Collection.displayField, id), + Map.of(COORDINATES, coordinate)) + ) .size(pageSize) ).build() : new Aggregation.Builder().composite(c -> c - .sources(List.of(Map.of(COORDINATES, coordinate), Map.of("id", id))) + .sources(List.of( + Map.of(StacBasicField.Collection.displayField, id), + Map.of(COORDINATES, coordinate)) + ) .size(pageSize) .after(afterKey) ).build(); @@ -449,12 +512,11 @@ public ElasticSearchBase.SearchResult searchFeatureSummary(String // Max value of field Aggregation max = MaxAggregation.of(s -> s.field(CQLFeatureFields.temporal.searchField))._toAggregation(); - // Field value of the group by + // Field value to return, think of it as select part of SQL Aggregation field = new Aggregation.Builder().topHits(th -> th.size(1) - .source(src -> src.filter(f -> f - .includes("geometry.geometry.coordinates")) - ) - .sort(createSortOptions(CQLFeatureFields.id.name(), CQLFeatureFields.class))) + .sort(createSortOptions( + String.format("%s,%s", CQLFeatureFields.collection.name(), CQLFeatureFields.geometry.name()), + CQLFeatureFields.class))) .build(); Aggregation aggregation = new Aggregation.Builder() From edc2340b67aa8690238eec75d757fcf195fe50a5 Mon Sep 17 00:00:00 2001 From: rng Date: Fri, 17 Jan 2025 14:42:30 +1100 Subject: [PATCH 08/10] Fix issue with aggregation --- .../server/core/mapper/StacToCollections.java | 2 +- .../core/mapper/StacToFeatureCollection.java | 18 ++-- .../model/enumeration/CQLFeatureFields.java | 12 +++ .../server/core/service/ElasticSearch.java | 73 +++++++++++----- .../core/service/ElasticSearchBase.java | 42 ++++++++-- .../org/aodn/ogcapi/server/BaseTestClass.java | 42 +++++++--- .../ogcapi/server/common/RestApiTest.java | 54 ++++++------ .../ogcapi/server/common/RestExtApiTest.java | 30 +++---- .../ogcapi/server/features/RestApiTest.java | 84 +++++++++++++++++-- .../aodn/ogcapi/server/tile/RestApiTest.java | 29 +++---- .../src/test/resources/data_index_schema.json | 46 ++++++++++ .../sample1.0.json | 31 +++++++ .../sample1.1.json | 31 +++++++ .../sample1.2.json | 31 +++++++ .../sample2.0.json | 31 +++++++ .../sample3.0.json | 31 +++++++ .../sample3.1.json | 31 +++++++ .../sample3.2.json | 31 +++++++ .../portal_records_index_schema.json | 6 +- 19 files changed, 536 insertions(+), 119 deletions(-) create mode 100644 server/src/test/resources/data_index_schema.json create mode 100644 server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json create mode 100644 server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.1.json create mode 100644 server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json create mode 100644 server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json create mode 100644 server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json create mode 100644 server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json create mode 100644 server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java index dad013c7..bd0c6004 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java @@ -23,7 +23,7 @@ public abstract class StacToCollections implements Converter model, Filter filter) { - List collections = model.getCollections().parallelStream() + List collections = model.getCollections().stream() .map(m -> getCollection(m, filter, hostname)) .collect(Collectors.toList()); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToFeatureCollection.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToFeatureCollection.java index c11c7c62..d580f5c1 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToFeatureCollection.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToFeatureCollection.java @@ -5,7 +5,6 @@ import au.org.aodn.ogcapi.features.model.PointGeoJSON; import au.org.aodn.ogcapi.server.core.model.StacItemModel; import au.org.aodn.ogcapi.server.core.service.ElasticSearch; -import com.fasterxml.jackson.core.type.TypeReference; import org.mapstruct.Mapper; import org.opengis.filter.Filter; import org.springframework.stereotype.Service; @@ -14,8 +13,6 @@ import java.util.List; import java.util.Map; -import static au.org.aodn.ogcapi.server.core.util.CommonUtils.safeGet; - @Service @Mapper(componentModel = "spring") public abstract class StacToFeatureCollection implements Converter, FeatureCollectionGeoJSON> { @@ -30,13 +27,16 @@ public FeatureCollectionGeoJSON convert(ElasticSearch.SearchResult ((Map>)i.getGeometry().get("geometry")).get("coordinates")) - .ifPresent(g -> - feature.setGeometry(new PointGeoJSON() - .type(PointGeoJSON.TypeEnum.POINT) - .coordinates(g) - )); + if(i.getGeometry().get("geometry") instanceof Map map) { + if(map.get("coordinates") instanceof List coords) { + List c = coords.stream() + .filter(item -> item instanceof BigDecimal) + .map(item -> (BigDecimal)item) + .toList(); + feature.setGeometry(new PointGeoJSON().coordinates(c)); + } + } feature.setProperties(i.getProperties()); return feature; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java index 703ed73c..b62b0a2f 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java @@ -45,6 +45,18 @@ public enum CQLFeatureFields implements CQLFieldsInterface { "geometry", null, (order) -> new SortOptions.Builder().field(f -> f.field("geometry.geometry.coordinates").order(order)) + ), + lat( + "properties.lat", + "properties.lat", + null, + (order) -> new SortOptions.Builder().field(f -> f.field("properties.lat").order(order)) + ), + lng( + "properties.lng", + "properties.lng", + null, + (order) -> new SortOptions.Builder().field(f -> f.field("properties.lng").order(order)) ); // Field that use to do sort, elastic search treat FieldData (searchField) differently, a searchField is not diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 95fd1b2f..0b62bd3b 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -28,8 +28,9 @@ import org.springframework.http.ResponseEntity; import java.io.IOException; +import java.math.BigDecimal; import java.util.*; -import java.util.function.Function; +import java.util.function.BiFunction; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -450,7 +451,10 @@ protected static FieldValue toFieldValue(String s) { * { * "coordinates": { * "terms": { - * "field": "geometry.geometry.coordinates" + * "script": { + * "source": "doc['geometry.geometry.coordinates'].value.toString()", + * "lang": "painless" + * } * } * } * } @@ -474,29 +478,40 @@ public ElasticSearchBase.SearchResult searchFeatureSummary(String final String MIN_TIME = "min_time"; final String MAX_TIME = "max_time"; - Function, SearchRequest.Builder> builderSupplier = (afterKey) -> { + BiFunction, Map, SearchRequest.Builder> builderSupplier = ( + arguments, afterKey) -> { + SearchRequest.Builder builder = new SearchRequest.Builder(); - // Group by coordinates - CompositeAggregationSource coordinate = CompositeAggregationSource.of(c -> c.terms(t -> t - .field("geometry.geometry.coordinates"))); - // Group by id - CompositeAggregationSource id = CompositeAggregationSource.of(c -> c.terms(t -> t - .field(StacBasicField.Collection.sortField))); - - // Use t page to another batch of records if exist - Aggregation groupBy = afterKey == null ? + + builder.query(q -> q + .term(t -> t + .field(CQLFeatureFields.collection.searchField) + .value(arguments.get("collectionId")) + ) + ); + + // Group by lng + CompositeAggregationSource lng = CompositeAggregationSource.of(c -> c.terms(t -> t + .field(CQLFeatureFields.lng.searchField))); + + // Group by lat + CompositeAggregationSource lat = CompositeAggregationSource.of(c -> c.terms(t -> t + .field(CQLFeatureFields.lat.searchField))); + + // Use afterKey to page to another batch of records if exist + Aggregation compose = afterKey == null ? new Aggregation.Builder().composite(c -> c .sources(List.of( - Map.of(StacBasicField.Collection.displayField, id), - Map.of(COORDINATES, coordinate)) + Map.of(CQLFeatureFields.lng.name(), lng), + Map.of(CQLFeatureFields.lat.name(), lat)) ) .size(pageSize) ).build() : new Aggregation.Builder().composite(c -> c .sources(List.of( - Map.of(StacBasicField.Collection.displayField, id), - Map.of(COORDINATES, coordinate)) + Map.of(CQLFeatureFields.lng.name(), lng), + Map.of(CQLFeatureFields.lat.name(), lat)) ) .size(pageSize) .after(afterKey) @@ -515,12 +530,12 @@ public ElasticSearchBase.SearchResult searchFeatureSummary(String // Field value to return, think of it as select part of SQL Aggregation field = new Aggregation.Builder().topHits(th -> th.size(1) .sort(createSortOptions( - String.format("%s,%s", CQLFeatureFields.collection.name(), CQLFeatureFields.geometry.name()), + String.format("%s,%s", CQLFeatureFields.lng.name(), CQLFeatureFields.lat.name()), CQLFeatureFields.class))) .build(); Aggregation aggregation = new Aggregation.Builder() - .composite(groupBy.composite()) + .composite(compose.composite()) .aggregations(Map.of( TOTAL_COUNT, sum, MIN_TIME, min, @@ -529,9 +544,17 @@ public ElasticSearchBase.SearchResult searchFeatureSummary(String )) .build(); + // There is a limitation that all sort field, assume to be inside the properties + Aggregation nested = new Aggregation.Builder().nested(n -> n + .path("properties") + ) + .aggregations(COORDINATES, aggregation) + .build(); + + builder.index(dataIndexName) .size(0) // Do not return hits, only aggregations, that is the hits().hit() section will be empty - .aggregations(COORDINATES, aggregation); + .aggregations(COORDINATES, nested); return builder; }; @@ -540,7 +563,11 @@ public ElasticSearchBase.SearchResult searchFeatureSummary(String ElasticSearchBase.SearchResult result = new ElasticSearchBase.SearchResult<>(); result.setCollections(new ArrayList<>()); - Iterable response = pageableAggregation(builderSupplier, CompositeBucket.class, null); + Map arguments = Map.of( + "collectionId", FieldValue.of(collectionId), + "aggKey", FieldValue.of(COORDINATES) + ); + Iterable response = pageableAggregation(builderSupplier, CompositeBucket.class, arguments, null); for (CompositeBucket node : response) { if (node != null) { @@ -554,7 +581,11 @@ public ElasticSearchBase.SearchResult searchFeatureSummary(String JsonData jd = th.hits().hits().get(0).source(); if(jd != null) { Map map = jd.to(Map.class); - model.geometry((Map) map.get(FeatureProperty.GEOMETRY.getValue())); + BigDecimal lng = BigDecimal.valueOf((double)map.get("lng")); + BigDecimal lat = BigDecimal.valueOf((double)map.get("lat")); + model.geometry(Map.of("geometry", Map.of( + "coordinates", List.of(lng, lat) + ))); } SumAggregate sa = node.aggregations().get(TOTAL_COUNT).sum(); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java index a52e9432..13824a58 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java @@ -29,7 +29,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Function; +import java.util.function.BiFunction; import java.util.function.Supplier; /** @@ -388,10 +388,12 @@ public Hit next() { * @param A generic type for Elastic query */ protected Iterable pageableAggregation( - Function, SearchRequest.Builder> requestBuilder, Class clazz, Long maxSize) { + BiFunction, Map, SearchRequest.Builder> requestBuilder, + Class clazz, + Map arguments, + Long maxSize) { try { - SearchRequest sr = requestBuilder.apply(null).build(); - String aggKey = sr.aggregations().keySet().stream().findFirst().orElse(""); + SearchRequest sr = requestBuilder.apply(arguments, null).build(); log.debug("Final elastic aggregation payload {}", sr); final AtomicLong count = new AtomicLong(0); @@ -409,7 +411,13 @@ public boolean hasNext() { return count.get() < maxSize; } - Aggregate ags = response.get().aggregations().get(aggKey); + Aggregate ags = response.get() + .aggregations() + .get(arguments.get("aggKey").stringValue()) + .nested() + .aggregations() + .get(arguments.get("aggKey").stringValue()); + Buckets stb = ags.composite().buckets(); // If we hit the end, that means we have iterated to end of page. @@ -424,13 +432,23 @@ public boolean hasNext() { try { // Get the last sorted value from the last batch // Use the last builder and append the searchAfter values - SearchRequest request = requestBuilder.apply(ags.composite().afterKey()).build(); + SearchRequest request = requestBuilder.apply(arguments, ags.composite().afterKey()).build(); log.debug("Final elastic aggregation payload {}", request.toString()); response.set(esClient.search(request, clazz)); // Reset counter from start index = 0; - return index < response.get().hits().hits().size(); + + ags = response.get() + .aggregations() + .get(arguments.get("aggKey").stringValue()) + .nested() + .aggregations() + .get(arguments.get("aggKey").stringValue()); + + stb = ags.composite().buckets(); + + return index < stb.array().size(); } catch(IOException ieo) { throw new RuntimeException(ieo); @@ -442,7 +460,13 @@ public boolean hasNext() { public T next() { count.incrementAndGet(); - Aggregate ags = response.get().aggregations().get(aggKey); + Aggregate ags = response.get() + .aggregations() + .get(arguments.get("aggKey").stringValue()) + .nested() + .aggregations() + .get(arguments.get("aggKey").stringValue()); + Buckets stb = ags.composite().buckets(); if(index < stb.array().size()) { @@ -454,7 +478,7 @@ public T next() { } }; } - catch(IOException e) { + catch(Exception e) { log.error("Fail to fetch record", e); } return Collections.emptySet(); diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java b/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java index 6c36dcea..f76d0f59 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java @@ -11,6 +11,8 @@ import co.elastic.clients.transport.rest_client.RestClientTransport; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.annotation.PostConstruct; +import lombok.extern.slf4j.Slf4j; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; @@ -33,6 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +@Slf4j public class BaseTestClass { @LocalServerPort @@ -53,9 +56,15 @@ public class BaseTestClass { @Value("${elasticsearch.index.name}") protected String record_index_name; + @Value("${elasticsearch.cloud_optimized_index.name}") + protected String co_data_index_name; + @Value("${elasticsearch.vocabs_index.name}") protected String vocabs_index_name; + @Value("${elasticsearch.cloud_optimized_index.name}") + protected String data_index_name; + protected Logger logger = LoggerFactory.getLogger(BaseTestClass.class); protected String getBasePath() { @@ -66,12 +75,18 @@ protected String getExternalBasePath() { return "http://localhost:" + port + "/api/v1/ogc/ext"; } - protected void clearElasticIndex() throws IOException { + protected List> schemas; - List> schemas = List.of( + @PostConstruct + public void initSchemas() { + schemas = List.of( + Map.of("name", record_index_name, "mapping", "portal_records_index_schema.json"), Map.of("name", vocabs_index_name, "mapping", "vocabs_index_schema.json"), - Map.of("name", record_index_name, "mapping", "portal_records_index_schema.json") + Map.of("name", data_index_name, "mapping", "data_index_schema.json") ); + } + + protected void clearElasticIndex() { logger.debug("Clear elastic index"); try { @@ -99,11 +114,6 @@ protected void clearElasticIndex() throws IOException { protected void createElasticIndex() { - List> schemas = List.of( - Map.of("name", record_index_name, "mapping", "portal_records_index_schema.json"), - Map.of("name", vocabs_index_name, "mapping", "vocabs_index_schema.json") - ); - schemas.forEach(schema -> { try { // TODO: This file should come from indexer jar when CodeArtifact in place @@ -119,7 +129,7 @@ protected void createElasticIndex() { // Ignore it, happens when index already created } } catch (IOException e) { - e.printStackTrace(); + log.error("Error", e); } }); } @@ -201,14 +211,14 @@ protected void bulkIndexVocabs(List vocabs) throws IOException { } } - protected void insertJsonToElasticIndex(String... filenames) throws IOException { + protected void insertJsonToElasticIndex(String index, String[] filenames) throws IOException { // Now insert json to index for(String filename : filenames) { File j = ResourceUtils.getFile("classpath:databag/" + filename); try(Reader reader = new FileReader(j)) { IndexResponse indexResponse = client.index(i -> i - .index(record_index_name) + .index(index) .withJson(reader)); logger.info("Sample file {}, indexed with response : {}", filename, indexResponse); @@ -221,7 +231,7 @@ protected void insertJsonToElasticIndex(String... filenames) throws IOException // Check the number of doc store inside the ES instance is correct SearchRequest.Builder b = new SearchRequest.Builder() - .index(record_index_name) + .index(index) .query(QueryBuilders.matchAll().build()._toQuery()); SearchRequest request = b.build(); @@ -233,6 +243,14 @@ protected void insertJsonToElasticIndex(String... filenames) throws IOException assertEquals(filenames.length, response.hits().hits().size(), "Number of docs stored is correct"); } + protected void insertJsonToElasticRecordIndex(String... filenames) throws IOException { + this.insertJsonToElasticIndex(record_index_name, filenames); + } + + protected void insertJsonToElasticCODataIndex(String... filenames) throws IOException { + this.insertJsonToElasticIndex(co_data_index_name, filenames); + } + protected Response getClusterHealth() throws IOException { return transport.restClient().performRequest(new Request("GET", "/_cluster/health")); } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java index fb2eb088..cfdaeec6 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java @@ -30,12 +30,12 @@ public void beforeClass() { } @AfterAll - public void clear() throws IOException { + public void clear() { super.clearElasticIndex(); } @BeforeEach - public void afterTest() throws IOException { + public void afterTest() { super.clearElasticIndex(); } @@ -44,13 +44,13 @@ public void verifyApiGet() { RestApi api = new RestApi(); ResponseEntity response = api.apiGet(OGCMediaTypeMapper.json.toString()); - assertEquals(response.getStatusCode(), HttpStatus.TEMPORARY_REDIRECT, "Incorrect redirect"); - assertEquals(Objects.requireNonNull(response.getHeaders().getLocation()).getPath() , "/api/v1/ogc/api-docs/v3", "Incorrect path"); + assertEquals(HttpStatus.TEMPORARY_REDIRECT, response.getStatusCode(), "Incorrect redirect"); + assertEquals("/api/v1/ogc/api-docs/v3", Objects.requireNonNull(response.getHeaders().getLocation()).getPath(), "Incorrect path"); response = api.apiGet(OGCMediaTypeMapper.html.toString()); - assertEquals(response.getStatusCode(), HttpStatus.TEMPORARY_REDIRECT, "Incorrect redirect"); - assertEquals(Objects.requireNonNull(response.getHeaders().getLocation()).getPath(), "/api/v1/ogc/swagger-ui/index.html", "Incorrect path"); + assertEquals(HttpStatus.TEMPORARY_REDIRECT, response.getStatusCode(), "Incorrect redirect"); + assertEquals("/api/v1/ogc/swagger-ui/index.html", Objects.requireNonNull(response.getHeaders().getLocation()).getPath(), "Incorrect path"); } @Test @@ -65,7 +65,7 @@ public void verifyClusterIsHealthy() throws IOException { */ @Test public void verifyApiCollectionsQueryOnText1() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" ); @@ -109,7 +109,7 @@ public void verifyApiCollectionsQueryOnText1() throws IOException { */ @Test public void verifyApiCollectionsQueryOnText2() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "073fde5a-bff3-1c1f-e053-08114f8c5588.json", "9fdb1eee-bc28-43a9-88c5-972324784837.json" @@ -133,7 +133,7 @@ public void verifyApiCollectionsQueryOnText2() throws IOException { */ @Test public void verifyDateTimeAfterBounds() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "5c418118-2581-4936-b6fd-d6bedfe74f62.json" @@ -171,7 +171,7 @@ public void verifyDateTimeAfterBounds() throws IOException { */ @Test public void verifyDateTimeBeforeBounds() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "5c418118-2581-4936-b6fd-d6bedfe74f62.json" @@ -197,7 +197,7 @@ public void verifyDateTimeBeforeBounds() throws IOException { */ @Test public void verifyDateTimeBetweenBounds() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "5c418118-2581-4936-b6fd-d6bedfe74f62.json" @@ -231,7 +231,7 @@ public void verifyDateTimeBetweenBounds() throws IOException { */ @Test public void verifyDateTimeBoundsWithDiscreteTime() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "caf7220a-19e0-4a7f-9af6-eade6c79a47a.json" // This one have two start/end @@ -272,7 +272,7 @@ public void verifyDateTimeBoundsWithDiscreteTime() throws IOException { */ @Test public void verifyPropertiesParameter() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json" ); @@ -292,7 +292,7 @@ public void verifyPropertiesParameter() throws IOException { */ @Test public void verifyCQLPropertyIsNullIsNotNull() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", // Provider null "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" // Provider not null ); @@ -317,7 +317,7 @@ public void verifyCQLPropertyIsNullIsNotNull() throws IOException { */ @Test public void verifyCQLPropertyEqualsOperation() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", // Provider null "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" // Provider is IMOS ); @@ -336,7 +336,7 @@ public void verifyCQLPropertyEqualsOperation() throws IOException { */ @Test public void verifyCQLPropertyAndOperation() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", // Provider null "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" // Provider is IMOS ); @@ -363,7 +363,7 @@ public void verifyCQLPropertyAndOperation() throws IOException { */ @Test public void verifyParameterTextSearchMatch() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", // Provider null "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" // Provider is IMOS ); @@ -380,7 +380,7 @@ public void verifyParameterTextSearchMatch() throws IOException { @Test public void verifyParameterParameterVocabsSearchMatch() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" @@ -408,7 +408,7 @@ public void verifyParameterParameterVocabsSearchMatch() throws IOException { */ @Test public void verifyCQLPropertyOrOperation() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", // Provider null "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" // Provider is IMOS ); @@ -437,7 +437,7 @@ public void verifyCQLPropertyOrOperation() throws IOException { */ @Test public void verifyCQLPropertyIntersectOperation() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "b299cdcd-3dee-48aa-abdd-e0fcdbb9cadc.json" ); // Intersect with this polygon @@ -465,7 +465,7 @@ public void verifyCQLPropertyIntersectOperation() throws IOException { */ @Test public void verifyCQLPropertyDatasetGroup() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "5c418118-2581-4936-b6fd-d6bedfe74f62.json", // Provider null "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" // Provider is IMOS ); @@ -484,7 +484,7 @@ public void verifyCQLPropertyDatasetGroup() throws IOException { */ @Test public void verifyCQLPropertyScore() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" @@ -495,8 +495,8 @@ public void verifyCQLPropertyScore() throws IOException { // Make sure OR not work as it didn't make sense to use or with setting ResponseEntity error = testRestTemplate.getForEntity(getBasePath() + "/collections?filter=score>=2 OR parameter_vocabs='wave'", ErrorResponse.class); - assertEquals(error.getStatusCode(), HttpStatus.INTERNAL_SERVER_ERROR); - assertEquals(Objects.requireNonNull(error.getBody()).getMessage(), "Or combine with query setting do not make sense", "correct error"); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, error.getStatusCode()); + assertEquals("Or combine with query setting do not make sense", Objects.requireNonNull(error.getBody()).getMessage(), "correct error"); // Lower score but the fuzzy is now with operator AND, therefore it will try to match all words 'dataset' and 'includes' with fuzzy collections = testRestTemplate.getForEntity(getBasePath() + "/collections?q='dataset includes'&filter=score>=1", Collections.class); @@ -514,7 +514,7 @@ public void verifyCQLPropertyScore() throws IOException { */ @Test public void verifyCQLFuzzyKey() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" @@ -548,7 +548,7 @@ public void verifyErrorMessageCreated() { */ @Test public void verifySortBy() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" @@ -585,7 +585,7 @@ public void verifySortBy() throws IOException { */ @Test public void verifySortByTemporalCorrect() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "073fde5a-bff3-1c1f-e053-08114f8c5588.json", "5c418118-2581-4936-b6fd-d6bedfe74f62.json", "19da2ce7-138f-4427-89de-a50c724f5f54.json", diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestExtApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestExtApiTest.java index 8061cf98..a391cc05 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestExtApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestExtApiTest.java @@ -24,17 +24,17 @@ public class RestExtApiTest extends BaseTestClass { @BeforeAll - public void beforeClass() throws IOException { + public void beforeClass() { super.createElasticIndex(); } @AfterAll - public void clear() throws IOException { + public void clear() { super.clearElasticIndex(); } @BeforeEach - public void afterTest() throws IOException { + public void afterTest() { super.clearElasticIndex(); } @@ -48,11 +48,11 @@ public void verifyClusterIsHealthy() throws IOException { /** * - * @throws IOException + * @throws IOException - not expect to throw */ @Test public void verifyApiResponseOnIncompleteInput() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" ); @@ -69,11 +69,11 @@ public void verifyApiResponseOnIncompleteInput() throws IOException { /** * - * @throws IOException + * @throws IOException - not expect to throw */ @Test public void verifyApiResponseOnCompleteInput() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" ); @@ -89,11 +89,11 @@ public void verifyApiResponseOnCompleteInput() throws IOException { /** * - * @throws IOException + * @throws IOException - not expect to throw */ @Test public void verifyApiResponseOnTypoInputNoParameterVocabsFilter() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" @@ -110,7 +110,7 @@ public void verifyApiResponseOnTypoInputNoParameterVocabsFilter() throws IOExcep @Test public void verifyApiResponseOnTypoInputSingleParameterVocabsFilter() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" @@ -129,7 +129,7 @@ public void verifyApiResponseOnTypoInputSingleParameterVocabsFilter() throws IOE @Test public void verifyApiResponseOnTypoInputMultipleParameterVocabsFilter() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" @@ -148,7 +148,7 @@ public void verifyApiResponseOnTypoInputMultipleParameterVocabsFilter() throws I @Test public void verifyApiResponseOnTypoInputMultipleParameterVocabsFilterNoResults() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" @@ -165,7 +165,7 @@ public void verifyApiResponseOnTypoInputMultipleParameterVocabsFilterNoResults() public void verifyApiResponseOnParameterVocabSuggestions() throws IOException { super.insertTestVocabs(); - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" ); @@ -192,7 +192,7 @@ public void verifyApiResponseOnParameterVocabSuggestions() throws IOException { public void verifyApiResponseOnPlatformVocabSuggestions() throws IOException { super.insertTestVocabs(); - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "record_with_parameter_platform_organisation_vocabs.json" ); @@ -238,7 +238,7 @@ public void verifyApiResponseOnPlatformVocabSuggestions() throws IOException { public void verifyApiResponseOnOrganisationVocabSuggestions() throws IOException { super.insertTestVocabs(); - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "record_with_parameter_platform_organisation_vocabs.json" ); diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 47fdbda9..722fd931 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -1,8 +1,12 @@ package au.org.aodn.ogcapi.server.features; import au.org.aodn.ogcapi.features.model.Collection; +import au.org.aodn.ogcapi.features.model.FeatureCollectionGeoJSON; +import au.org.aodn.ogcapi.features.model.FeatureGeoJSON; +import au.org.aodn.ogcapi.features.model.PointGeoJSON; import au.org.aodn.ogcapi.server.BaseTestClass; import au.org.aodn.ogcapi.server.core.model.ExtendedCollections; +import au.org.aodn.ogcapi.server.core.model.enumeration.FeatureProperty; import org.junit.jupiter.api.*; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; @@ -33,12 +37,12 @@ public void beforeClass() { } @AfterAll - public void clear() throws IOException { + public void clear() { super.clearElasticIndex(); } @BeforeEach - public void afterTest() throws IOException { + public void afterTest() { super.clearElasticIndex(); } @@ -57,7 +61,7 @@ public void verifyCorrectInternalPagingLargeData() throws IOException { // Given 6 records and we set page to 4, that means each query elastic return 4 record only // and the logic to load the reset can kick in. - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "5c418118-2581-4936-b6fd-d6bedfe74f62.json", "19da2ce7-138f-4427-89de-a50c724f5f54.json", "516811d7-cd1e-207a-e0440003ba8c79dd.json", @@ -99,7 +103,7 @@ public void verifyCorrectPageSizeDataReturn() throws IOException { // Given 6 records and we set page to 4, that means each query elastic return 4 record only // and the logic to load the reset can kick in. - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "5c418118-2581-4936-b6fd-d6bedfe74f62.json", "19da2ce7-138f-4427-89de-a50c724f5f54.json", "516811d7-cd1e-207a-e0440003ba8c79dd.json", @@ -187,7 +191,7 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { // Given 6 records and we set page to 4, that means each query elastic return 4 record only // and the logic to load the reset can kick in. - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "5c418118-2581-4936-b6fd-d6bedfe74f62.json", "19da2ce7-138f-4427-89de-a50c724f5f54.json", "516811d7-cd1e-207a-e0440003ba8c79dd.json", @@ -288,7 +292,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { // Given 6 records and we set page to 4, that means each query elastic return 4 record only // and the logic to load the reset can kick in. - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "5c418118-2581-4936-b6fd-d6bedfe74f62.json", "19da2ce7-138f-4427-89de-a50c724f5f54.json", "516811d7-cd1e-207a-e0440003ba8c79dd.json", @@ -358,7 +362,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { @Test public void verifyGetSingleCollection() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" ); @@ -377,7 +381,7 @@ public void verifyGetSingleCollection() throws IOException { @Test public void verifyBBoxCorrect() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "ae86e2f5-eaaf-459e-a405-e654d85adb9c.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" ); @@ -430,5 +434,69 @@ public void verifyBBoxCorrect() throws IOException { assertEquals(154.0, bbox.get(0).get(2).doubleValue(), "Overall bounding box coor 3"); assertEquals(-9.0, bbox.get(0).get(3).doubleValue(), "Overall bounding box coor 4"); } + /** + * Verify the function correctly sum up the values for feature id summary + * @throws IOException - Not expect to throw + */ + @Test + public void verifyAggregationFeatureSummaryCorrect() throws IOException { + super.insertJsonToElasticCODataIndex( + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.1.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json" + ); + + // Call rest api directly and get query result + ResponseEntity collection = testRestTemplate.getForEntity( + getBasePath() + "/collections/35234913-aa3c-48ec-b9a4-77f822f66ef8/items/summary", + FeatureCollectionGeoJSON.class); + + assertNotNull(collection.getBody(), "Body not null"); + + FeatureCollectionGeoJSON json = collection.getBody(); + assertEquals(3, json.getFeatures().size(), "Features correct"); + + // Sort make sure compare always same order + List sf = json.getFeatures().stream() + .sorted((a,b) -> b.getGeometry().hashCode() - a.getGeometry().hashCode()) + .toList(); + // Sample1 + FeatureGeoJSON featureGeoJSON1 = new FeatureGeoJSON(); + featureGeoJSON1.setType(FeatureGeoJSON.TypeEnum.FEATURE); + featureGeoJSON1.setGeometry(new PointGeoJSON().coordinates(List.of(BigDecimal.valueOf(159.26), BigDecimal.valueOf(-24.72)))); + featureGeoJSON1.setProperties(Map.of( + FeatureProperty.COUNT.getValue(), 42.0, + FeatureProperty.START_TIME.getValue(), "2023-02-01T00:00:00.000Z", + FeatureProperty.END_TIME.getValue(), "2023-02-01T00:00:00.000Z" + )); + assertEquals(featureGeoJSON1, sf.get(0), "featureGeoJSON1"); + + // Sample3 + FeatureGeoJSON featureGeoJSON2 = new FeatureGeoJSON(); + featureGeoJSON2.setType(FeatureGeoJSON.TypeEnum.FEATURE); + featureGeoJSON2.setGeometry(new PointGeoJSON().coordinates(List.of(BigDecimal.valueOf(154.81), BigDecimal.valueOf(-26.2)))); + featureGeoJSON2.setProperties(Map.of( + FeatureProperty.COUNT.getValue(), 48.0, + FeatureProperty.START_TIME.getValue(), "2023-02-01T00:00:00.000Z", + FeatureProperty.END_TIME.getValue(), "2024-03-01T00:00:00.000Z" + + )); + assertEquals(featureGeoJSON2, sf.get(1), "featureGeoJSON2"); + + FeatureGeoJSON featureGeoJSON3 = new FeatureGeoJSON(); + featureGeoJSON3.setType(FeatureGeoJSON.TypeEnum.FEATURE); + featureGeoJSON3.setGeometry(new PointGeoJSON().coordinates(List.of(BigDecimal.valueOf(153.56), BigDecimal.valueOf(-26.59)))); + featureGeoJSON3.setProperties(Map.of( + FeatureProperty.COUNT.getValue(), 14.0, + FeatureProperty.START_TIME.getValue(), "2023-02-01T00:00:00.000Z", + FeatureProperty.END_TIME.getValue(), "2023-02-01T00:00:00.000Z" + + )); + assertEquals(featureGeoJSON3, sf.get(2), "featureGeoJSON3"); + } } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/tile/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/tile/RestApiTest.java index 4d7366e6..5efdb2ff 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/tile/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/tile/RestApiTest.java @@ -9,9 +9,6 @@ import java.io.IOException; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles("test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @@ -19,17 +16,17 @@ public class RestApiTest extends BaseTestClass { @BeforeAll - public void beforeClass() throws IOException { + public void beforeClass() { super.createElasticIndex(); } @AfterAll - public void clear() throws IOException { + public void clear() { super.clearElasticIndex(); } @BeforeEach - public void afterTest() throws IOException { + public void afterTest() { super.clearElasticIndex(); } @@ -43,7 +40,7 @@ public void verifyClusterIsHealthy() throws IOException { */ @Test public void verifyTiles() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" ); @@ -53,16 +50,18 @@ public void verifyTiles() throws IOException { InlineResponse2002.class ); - assertEquals(2, tiles.getBody().getTilesets().size(), "Count is correct"); - assertEquals("Impacts of stress on coral reproduction.", tiles.getBody().getTilesets().get(0).getTitle(), "Title matched 1"); - assertEquals("Ocean acidification historical reconstruction", tiles.getBody().getTilesets().get(1).getTitle(), "Title matched 2"); + Assertions.assertNotNull(tiles.getBody(), "Body not null"); + Assertions.assertNotNull(tiles.getBody().getTilesets(), "Tilesets not null"); + Assertions.assertEquals(2, tiles.getBody().getTilesets().size(), "Count is correct"); + Assertions.assertEquals("Impacts of stress on coral reproduction.", tiles.getBody().getTilesets().get(0).getTitle(), "Title matched 1"); + Assertions.assertEquals("Ocean acidification historical reconstruction", tiles.getBody().getTilesets().get(1).getTitle(), "Title matched 2"); } /** * Verify api call /collections/{collectionId}/tiles */ @Test public void verifyCollectionTiles() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" ); @@ -72,8 +71,10 @@ public void verifyCollectionTiles() throws IOException { InlineResponse2002.class ); - assertEquals(1, tiles.getBody().getTilesets().size(), "Count is correct"); - assertEquals("Impacts of stress on coral reproduction.", tiles.getBody().getTilesets().get(0).getTitle(), "Title matched 1"); + Assertions.assertNotNull(tiles.getBody(), "Body not null"); + Assertions.assertNotNull(tiles.getBody().getTilesets(), "Tilesets not null"); + Assertions.assertEquals(1, tiles.getBody().getTilesets().size(), "Count is correct"); + Assertions.assertEquals("Impacts of stress on coral reproduction.", tiles.getBody().getTilesets().get(0).getTitle(), "Title matched 1"); } /** * Verify api call /tiles/{tileMatrixSetId}/{tileMatrix}/{tileRow}/{tileCol}, this call will return the bounding @@ -82,7 +83,7 @@ public void verifyCollectionTiles() throws IOException { */ @Test public void verifyTilesMatrixSetXYZ() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "b299cdcd-3dee-48aa-abdd-e0fcdbb9cadc.json" ); diff --git a/server/src/test/resources/data_index_schema.json b/server/src/test/resources/data_index_schema.json new file mode 100644 index 00000000..28ad4c00 --- /dev/null +++ b/server/src/test/resources/data_index_schema.json @@ -0,0 +1,46 @@ +{ + "mappings": { + "properties": { + "id": { + "type": "keyword" + }, + "stac_version": { "type": "text" }, + "stac_extensions": { "type": "text" }, + "type": { "type": "text" }, + "collection": { + "type": "keyword" + }, + "geometry": { + "type": "geo_shape" + }, + "bbox" : { + "type": "double" + }, + "properties" : { + "type": "nested", + "properties" : { + "lng": { "type": "double" }, + "lat": { "type": "double" }, + "depth": { "type": "double" }, + "count": { "type": "double" }, + "time": { "type": "date" } + } + }, + "links": { + "type": "nested", + "properties": { + "link" : { + "type": "nested", + "properties": { + "href": { "type": "text" }, + "rel": { "type": "text" }, + "type": { "type": "text" }, + "title": { "type": "text" }, + "description": { "type": "text" } + } + } + } + } + } + } +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json new file mode 100644 index 00000000..cbf960b8 --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|159.26|-24.72|150.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 159.26, + -24.72 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 150, + "lng": 159.26, + "lat": -24.72, + "time": "2023-02-01T00:00:00Z", + "count": 14 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} \ No newline at end of file diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.1.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.1.json new file mode 100644 index 00000000..985967bf --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.1.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|159.26|-24.72|940.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 159.26, + -24.72 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 940, + "lng": 159.26, + "lat": -24.72, + "time": "2023-02-01T00:00:00Z", + "count": 13 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json new file mode 100644 index 00000000..191394b0 --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|159.26|-24.72|570.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 159.26, + -24.72 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 570, + "lng": 159.26, + "lat": -24.72, + "time": "2023-02-01T00:00:00Z", + "count": 15 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} \ No newline at end of file diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json new file mode 100644 index 00000000..792c48dd --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|153.56|-26.59|760.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 153.56, + -26.59 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 760, + "lng": 153.56, + "lat": -26.59, + "time": "2023-02-01T00:00:00Z", + "count": 14 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} \ No newline at end of file diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json new file mode 100644 index 00000000..780dfb07 --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|154.81|-26.20|260.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 154.81, + -26.2 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 260, + "lng": 154.81, + "lat": -26.2, + "time": "2023-02-01T00:00:00Z", + "count": 14 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} \ No newline at end of file diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json new file mode 100644 index 00000000..b93bde72 --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2024-03|154.81|-26.20|260.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 154.81, + -26.2 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 260, + "lng": 154.81, + "lat": -26.2, + "time": "2024-03-01T00:00:00Z", + "count": 20 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} \ No newline at end of file diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json new file mode 100644 index 00000000..47e13ee7 --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|154.81|-26.20|260.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 154.81, + -26.2 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 260, + "lng": 154.81, + "lat": -26.2, + "time": "2023-02-01T00:00:00Z", + "count": 14 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} \ No newline at end of file diff --git a/server/src/test/resources/portal_records_index_schema.json b/server/src/test/resources/portal_records_index_schema.json index e85f05d4..1600c8c3 100644 --- a/server/src/test/resources/portal_records_index_schema.json +++ b/server/src/test/resources/portal_records_index_schema.json @@ -88,6 +88,9 @@ "organisation_vocabs_sayt": { "type": "search_as_you_type", "analyzer": "custom_analyser" } } }, + "parameter_vocabs": { "type": "keyword" }, + "platform_vocabs": { "type": "keyword" }, + "organisation_vocabs": { "type": "keyword" }, "keywords": { "type": "nested", "properties": { @@ -162,9 +165,6 @@ } } }, - "parameter_vocabs": { "type": "keyword" }, - "platform_vocabs": { "type": "keyword" }, - "organisation_vocabs": { "type": "keyword" }, "statement": { "type": "text" } } }, From 807a061f29d1129555e5ad6334ef1607c9cffae4 Mon Sep 17 00:00:00 2001 From: rng Date: Fri, 17 Jan 2025 15:03:52 +1100 Subject: [PATCH 09/10] Add one more test case to cover paging --- .../ogcapi/server/features/RestApiTest.java | 60 +++++++++++++++++++ .../sample4.0.json | 31 ++++++++++ .../sample5.0.json | 31 ++++++++++ .../sample5.1.json | 31 ++++++++++ 4 files changed, 153 insertions(+) create mode 100644 server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample4.0.json create mode 100644 server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.0.json create mode 100644 server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.1.json diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 722fd931..0af66a61 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -499,4 +499,64 @@ public void verifyAggregationFeatureSummaryCorrect() throws IOException { )); assertEquals(featureGeoJSON3, sf.get(2), "featureGeoJSON3"); } + /** + * We add more sample data and will trigger page load. + * @throws IOException - Not expect to throw + */ + @Test + public void verifyAggregationFeatureSummaryWithPageCorrect() throws IOException { + assertEquals(4, pageSize, "This test only works with small page"); + + super.insertJsonToElasticCODataIndex( + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.1.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample4.0.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.0.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.1.json" + ); + + // Call rest api directly and get query result + ResponseEntity collection = testRestTemplate.getForEntity( + getBasePath() + "/collections/35234913-aa3c-48ec-b9a4-77f822f66ef8/items/summary", + FeatureCollectionGeoJSON.class); + + assertNotNull(collection.getBody(), "Body not null"); + + FeatureCollectionGeoJSON json = collection.getBody(); + assertEquals(5, json.getFeatures().size(), "Features correct"); + + // Sort make sure compare always same order + List sf = json.getFeatures().stream() + .sorted((a,b) -> b.getGeometry().hashCode() - a.getGeometry().hashCode()) + .toList(); + + // Sample1 + FeatureGeoJSON featureGeoJSON1 = new FeatureGeoJSON(); + featureGeoJSON1.setType(FeatureGeoJSON.TypeEnum.FEATURE); + featureGeoJSON1.setGeometry(new PointGeoJSON().coordinates(List.of(BigDecimal.valueOf(163.56), BigDecimal.valueOf(-26.59)))); + featureGeoJSON1.setProperties(Map.of( + FeatureProperty.COUNT.getValue(), 14.0, + FeatureProperty.START_TIME.getValue(), "2023-02-01T00:00:00.000Z", + FeatureProperty.END_TIME.getValue(), "2023-02-01T00:00:00.000Z" + + )); + assertEquals(featureGeoJSON1, sf.get(0), "featureGeoJSON1"); + + // Sample5 + FeatureGeoJSON featureGeoJSON2 = new FeatureGeoJSON(); + featureGeoJSON2.setType(FeatureGeoJSON.TypeEnum.FEATURE); + featureGeoJSON2.setGeometry(new PointGeoJSON().coordinates(List.of(BigDecimal.valueOf(163.56), BigDecimal.valueOf(-126.59)))); + featureGeoJSON2.setProperties(Map.of( + FeatureProperty.COUNT.getValue(), 20.0, + FeatureProperty.START_TIME.getValue(), "2022-12-01T00:00:00.000Z", + FeatureProperty.END_TIME.getValue(), "2023-02-01T00:00:00.000Z" + + )); + assertEquals(featureGeoJSON2, sf.get(1), "featureGeoJSON2"); + } } diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample4.0.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample4.0.json new file mode 100644 index 00000000..d883f065 --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample4.0.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|163.56|-26.59|760.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 163.56, + -26.59 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 760, + "lng": 163.56, + "lat": -26.59, + "time": "2023-02-01T00:00:00Z", + "count": 14 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} \ No newline at end of file diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.0.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.0.json new file mode 100644 index 00000000..4c8cf6e5 --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.0.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|163.56|-126.59|760.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 163.56, + -126.59 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 760, + "lng": 163.56, + "lat": -126.59, + "time": "2023-02-01T00:00:00Z", + "count": 10 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} \ No newline at end of file diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.1.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.1.json new file mode 100644 index 00000000..25156b8e --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.1.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|163.56|-126.59|760.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 163.56, + -126.59 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 760, + "lng": 163.56, + "lat": -126.59, + "time": "2022-12-01T00:00:00Z", + "count": 10 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} \ No newline at end of file From 7e19beb9296ed7b50a32f50f3235505d4e0eed25 Mon Sep 17 00:00:00 2001 From: rng Date: Fri, 17 Jan 2025 15:04:39 +1100 Subject: [PATCH 10/10] Pre-commit fix --- .../35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json | 2 +- .../35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json | 2 +- .../35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json | 2 +- .../35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json | 2 +- .../35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json | 2 +- .../35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json | 2 +- .../35234913-aa3c-48ec-b9a4-77f822f66ef8/sample4.0.json | 2 +- .../35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.0.json | 2 +- .../35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.1.json | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json index cbf960b8..f4e3a328 100644 --- a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json @@ -28,4 +28,4 @@ "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" ], "type": "Feature" -} \ No newline at end of file +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json index 191394b0..1c004c5b 100644 --- a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json @@ -28,4 +28,4 @@ "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" ], "type": "Feature" -} \ No newline at end of file +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json index 792c48dd..e9bf727e 100644 --- a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json @@ -28,4 +28,4 @@ "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" ], "type": "Feature" -} \ No newline at end of file +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json index 780dfb07..1653dbf4 100644 --- a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json @@ -28,4 +28,4 @@ "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" ], "type": "Feature" -} \ No newline at end of file +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json index b93bde72..b32c7662 100644 --- a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json @@ -28,4 +28,4 @@ "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" ], "type": "Feature" -} \ No newline at end of file +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json index 47e13ee7..071b94d8 100644 --- a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json @@ -28,4 +28,4 @@ "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" ], "type": "Feature" -} \ No newline at end of file +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample4.0.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample4.0.json index d883f065..cc8e5756 100644 --- a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample4.0.json +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample4.0.json @@ -28,4 +28,4 @@ "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" ], "type": "Feature" -} \ No newline at end of file +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.0.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.0.json index 4c8cf6e5..1b05f1e4 100644 --- a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.0.json +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.0.json @@ -28,4 +28,4 @@ "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" ], "type": "Feature" -} \ No newline at end of file +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.1.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.1.json index 25156b8e..f2e946f5 100644 --- a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.1.json +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.1.json @@ -28,4 +28,4 @@ "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" ], "type": "Feature" -} \ No newline at end of file +}