diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/common/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/common/RestApi.java index 89a74a62..fc32b357 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/common/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/common/RestApi.java @@ -9,6 +9,7 @@ import au.org.aodn.ogcapi.features.model.Exception; import au.org.aodn.ogcapi.server.core.mapper.StacToCollections; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType; +import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFields; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFilterType; import au.org.aodn.ogcapi.server.core.model.enumeration.OGCMediaTypeMapper; import au.org.aodn.ogcapi.server.core.service.OGCApiService; @@ -154,7 +155,10 @@ public ResponseEntity getCollections( // TODO: Support other CRS. if (CQLFilterType.convert(filterLang) == CQLFilterType.CQL && CQLCrsType.convertFromUrl(crs) == CQLCrsType.EPSG4326) { if (datetime != null) { - filter = OGCApiService.processDatetimeParameter(datetime, filter); + filter = OGCApiService.processDatetimeParameter(CQLFields.temporal.name(), datetime, filter); + } + if (bbox != null) { + filter = OGCApiService.processBBoxParameter(CQLFields.geometry.name(), bbox, filter); } return commonService.getCollectionList( q, diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java index acf3fb3f..bd0c6004 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java @@ -3,6 +3,7 @@ import au.org.aodn.ogcapi.features.model.Collection; import au.org.aodn.ogcapi.features.model.Collections; import au.org.aodn.ogcapi.server.core.model.ExtendedCollections; +import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; import au.org.aodn.ogcapi.server.core.service.ElasticSearch; import org.mapstruct.Mapper; import org.opengis.filter.Filter; @@ -14,15 +15,15 @@ @Service @Mapper(componentModel = "spring") -public abstract class StacToCollections implements Converter { +public abstract class StacToCollections implements Converter, Collections> { @Value("${api.host}") protected String hostname; @Override - public Collections convert(ElasticSearch.SearchResult model, Filter filter) { + public Collections convert(ElasticSearch.SearchResult model, Filter filter) { - List collections = model.getCollections().parallelStream() + List collections = model.getCollections().stream() .map(m -> getCollection(m, filter, hostname)) .collect(Collectors.toList()); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToFeatureCollection.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToFeatureCollection.java new file mode 100644 index 00000000..d580f5c1 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToFeatureCollection.java @@ -0,0 +1,50 @@ +package au.org.aodn.ogcapi.server.core.mapper; + +import au.org.aodn.ogcapi.features.model.FeatureCollectionGeoJSON; +import au.org.aodn.ogcapi.features.model.FeatureGeoJSON; +import au.org.aodn.ogcapi.features.model.PointGeoJSON; +import au.org.aodn.ogcapi.server.core.model.StacItemModel; +import au.org.aodn.ogcapi.server.core.service.ElasticSearch; +import org.mapstruct.Mapper; +import org.opengis.filter.Filter; +import org.springframework.stereotype.Service; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; + +@Service +@Mapper(componentModel = "spring") +public abstract class StacToFeatureCollection implements Converter, FeatureCollectionGeoJSON> { + + @Override + public FeatureCollectionGeoJSON convert(ElasticSearch.SearchResult model, Filter filter) { + FeatureCollectionGeoJSON f = new FeatureCollectionGeoJSON(); + f.setType(FeatureCollectionGeoJSON.TypeEnum.FEATURECOLLECTION); + + List features = model.getCollections().stream() + .map(i -> { + FeatureGeoJSON feature = new FeatureGeoJSON(); + feature.setType(FeatureGeoJSON.TypeEnum.FEATURE); + + if(i.getGeometry().get("geometry") instanceof Map map) { + if(map.get("coordinates") instanceof List coords) { + List c = coords.stream() + .filter(item -> item instanceof BigDecimal) + .map(item -> (BigDecimal)item) + .toList(); + + feature.setGeometry(new PointGeoJSON().coordinates(c)); + } + } + feature.setProperties(i.getProperties()); + + return feature; + }) + .filter(i -> i.getGeometry() != null) + .toList(); + + f.setFeatures(features); + return f; + } +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToInlineResponse2002.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToInlineResponse2002.java index 6a56347a..af55aefc 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToInlineResponse2002.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToInlineResponse2002.java @@ -1,5 +1,6 @@ package au.org.aodn.ogcapi.server.core.mapper; +import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType; import au.org.aodn.ogcapi.server.core.service.ElasticSearch; import au.org.aodn.ogcapi.tile.model.InlineResponse2002; @@ -14,13 +15,13 @@ @Service @Mapper(componentModel = "spring") -public abstract class StacToInlineResponse2002 implements Converter { +public abstract class StacToInlineResponse2002 implements Converter, InlineResponse2002> { @Value("${api.host}") protected String hostname; @Override - public InlineResponse2002 convert(ElasticSearch.SearchResult model, Filter noUse) { + public InlineResponse2002 convert(ElasticSearch.SearchResult model, Filter noUse) { List items = model.getCollections().stream() .map(m -> { TileSetItem item = new TileSetItem(); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToTileSetWmWGS84Q.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToTileSetWmWGS84Q.java index 637463a7..deeaddcb 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToTileSetWmWGS84Q.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToTileSetWmWGS84Q.java @@ -12,7 +12,7 @@ @Service @Mapper(componentModel = "spring") -public abstract class StacToTileSetWmWGS84Q implements Converter { +public abstract class StacToTileSetWmWGS84Q implements Converter, TileSet> { @Value("${api.host}") protected String hostname; @@ -27,7 +27,7 @@ protected static class TileSetWorldMercatorWGS84Quad extends TileSet { } @Override - public TileSet convert(ElasticSearch.SearchResult from, Filter noUse) { + public TileSet convert(ElasticSearch.SearchResult from, Filter noUse) { TileSetWorldMercatorWGS84Quad tileSet = new TileSetWorldMercatorWGS84Quad(); tileSet.setTileMatrixSetURI("http://www.opengis.net/def/tilematrixset/OGC/1.0/WorldMercatorWGS84Quad"); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/AssetModel.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/AssetModel.java new file mode 100644 index 00000000..1aa6a6fd --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/AssetModel.java @@ -0,0 +1,48 @@ +package au.org.aodn.ogcapi.server.core.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +@JsonInclude(JsonInclude.Include.NON_NULL) +public class AssetModel { + // https://github.com/radiantearth/stac-spec/blob/master/best-practices.md#list-of-asset-roles + public enum Role { + DATA("data"), + METADATA("metadata"), + THUMBNAIL("thumbnail"), + OVERVIEW("overview"), + VISUAL("visual"), + DATE("date"), + GRAPHIC("graphic"), + DATA_MASK("data-mask"), + SNOW_ICE("snow-ice"), + LAND_WATER("land-water"), + WATER_MASK("water-mask"), + ISO_19115("iso-19115"); + + private final String role; + + Role(String role) { + this.role = role; + } + + @Override + public String toString() { + return role; + } + } + /** + * REQUIRED. URI to the asset object. Relative and absolute URI are both allowed. Trailing slashes are significant. + */ + protected String href; + protected String title; + protected String description; + protected String type; + /** + * The semantic roles of the asset, similar to the use of rel in links. + */ + protected Role role; +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetSearchResult.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DataSearchResult.java similarity index 56% rename from server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetSearchResult.java rename to server/src/main/java/au/org/aodn/ogcapi/server/core/model/DataSearchResult.java index 37e0a1c0..0a7c93b7 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetSearchResult.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DataSearchResult.java @@ -9,13 +9,16 @@ import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; -public class DatasetSearchResult { +import static au.org.aodn.ogcapi.server.core.util.CommonUtils.safeGet; + +public class DataSearchResult { private final FeatureCollectionGeoJSON dataset; - public DatasetSearchResult() { + public DataSearchResult() { this.dataset = new FeatureCollectionGeoJSON(); initDataset(); } @@ -31,31 +34,48 @@ public FeatureCollectionGeoJSON getSummarizedDataset() { return summarizer.getSummarizedDataset(); } - public void addDatum(DatumModel datum) { + public void addDatum(StacItemModel datum) { if (datum == null) { throw new IllegalArgumentException("Datum cannot be null"); } - var feature = new FeatureGeoJSON(); + FeatureGeoJSON feature = new FeatureGeoJSON(); feature.setType(FeatureGeoJSON.TypeEnum.FEATURE); + var geometry = new PointGeoJSON(); geometry.setType(PointGeoJSON.TypeEnum.POINT); - var coordinates = new ArrayList(); + + List coordinates = new ArrayList<>(); // Don't use null checks here because it is a list and even if it is null, // it still needs "null" to occupy the space - coordinates.add(datum.getLongitude()); - coordinates.add(datum.getLatitude()); + safeGet(() -> ((Map)datum.getGeometry().get("geometry")).get("coordinates")) + .filter(item -> item instanceof List) + .map(item -> (List)item) + .ifPresent(item -> { + if(item.size() == 2) { + coordinates.add(new BigDecimal(item.get(0).toString())); + coordinates.add(new BigDecimal(item.get(1).toString())); + } + }); + geometry.setCoordinates(coordinates); feature.setGeometry(geometry); // Please add more properties if needed Map properties = new HashMap<>(); - properties.put(FeatureProperty.TIME.getValue(), datum.getTime()); - properties.put(FeatureProperty.DEPTH.getValue(), datum.getDepth()); - properties.put(FeatureProperty.COUNT.getValue(), datum.getCount()); + + safeGet(() -> ((Map)datum.getGeometry().get("properties")).get("depth")) + .ifPresent(val -> properties.put(FeatureProperty.DEPTH.getValue(), val)); + + safeGet(() -> (datum.getProperties().get("time"))) + .ifPresent(val -> properties.put(FeatureProperty.TIME.getValue(), val)); + + safeGet(() -> (datum.getProperties().get("count"))) + .ifPresent(val -> properties.put(FeatureProperty.COUNT.getValue(), val)); + feature.setProperties(properties); dataset.getFeatures().add(feature); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetModel.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetModel.java deleted file mode 100644 index 7aa62d63..00000000 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetModel.java +++ /dev/null @@ -1,12 +0,0 @@ -package au.org.aodn.ogcapi.server.core.model; - -import lombok.Builder; -import lombok.Data; - -import java.util.List; -@Data -@Builder -public class DatasetModel { - private String uuid; - private List data; -} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatumModel.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatumModel.java deleted file mode 100644 index 62be0a23..00000000 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatumModel.java +++ /dev/null @@ -1,18 +0,0 @@ -package au.org.aodn.ogcapi.server.core.model; - -import lombok.Builder; -import lombok.Data; - -import java.math.BigDecimal; - -@Data -@Builder -public class DatumModel { - - private String time; - private BigDecimal latitude; - private BigDecimal longitude; - private BigDecimal depth; - - private long count; -} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/StacItemModel.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/StacItemModel.java new file mode 100644 index 00000000..ebc19375 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/StacItemModel.java @@ -0,0 +1,76 @@ +package au.org.aodn.ogcapi.server.core.model; + +import au.org.aodn.ogcapi.features.model.FeatureCollectionGeoJSON; +import au.org.aodn.ogcapi.features.model.FeatureGeoJSON; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class StacItemModel { + + @JsonProperty("type") + protected String type; + /** + * REQUIRED. Provider identifier. The ID should be unique within the Collection that contains the Item. + */ + @JsonProperty("id") + protected String uuid; + /** + * REQUIRED. Defines the full footprint of the asset represented by this item, formatted according to RFC 7946, + * section 3.1 if a geometry is provided or section 3.2 if no geometry is provided. + * Use to generate the vector tile, the STAC format is not optimized and hard to work with for Elastic search + */ + @JsonProperty("geometry") + protected Map geometry; + /** + * REQUIRED if geometry is not null, prohibited if geometry is null. Bounding Box of the asset represented by + * this Item, formatted according to RFC 7946, section 5. + */ + @JsonProperty("bbox") + protected List> bbox; + /** + * REQUIRED. A dictionary of additional metadata for the Item. + */ + @JsonProperty("properties") + protected Map properties; + /** + * REQUIRED. List of link objects to resources and related URLs. See the best practices for details on when the + * use self links is strongly recommended. + */ + protected List links; + + protected Map assets; + /** + * The id of the STAC Collection this Item references to. This field is required if a link with a collection relation type is present and is not allowed otherwise. + */ + @JsonProperty("collection") + protected String collection; + + @JsonProperty("stac_version") + protected String stacVersion; + + @JsonProperty("stac_extensions") + public String[] getStacExtension() { + return new String[] { + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + }; + } + +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java new file mode 100644 index 00000000..b62b0a2f --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java @@ -0,0 +1,146 @@ +package au.org.aodn.ogcapi.server.core.model.enumeration; + +import co.elastic.clients.elasticsearch._types.SortOptions; +import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch._types.TopLeftBottomRightGeoBounds; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.util.ObjectBuilder; +import lombok.Getter; + +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Field name for cloud optimized data index + */ +public enum CQLFeatureFields implements CQLFieldsInterface { + + id( + StacBasicField.UUID.searchField, + StacBasicField.UUID.displayField, + null, + (order) -> new SortOptions.Builder().field(f -> f.field(StacBasicField.UUID.sortField).order(order)) + ), + collection( + StacBasicField.Collection.searchField, + StacBasicField.Collection.displayField, + null, + (order) -> new SortOptions.Builder().field(f -> f.field(StacBasicField.Collection.sortField).order(order)) + ), + temporal( + "properties.time", + "properties.time", + null, + null + ), + count( + "properties.count", + "properties.count", + null, + null + ), + geometry( + "geometry", + "geometry", + null, + (order) -> new SortOptions.Builder().field(f -> f.field("geometry.geometry.coordinates").order(order)) + ), + lat( + "properties.lat", + "properties.lat", + null, + (order) -> new SortOptions.Builder().field(f -> f.field("properties.lat").order(order)) + ), + lng( + "properties.lng", + "properties.lng", + null, + (order) -> new SortOptions.Builder().field(f -> f.field("properties.lng").order(order)) + ); + + // Field that use to do sort, elastic search treat FieldData (searchField) differently, a searchField is not + // efficient for sorting. + public final String searchField; // Field in STAC object + + @Getter + private final List displayField; + + // null value indicate it cannot be sort by that field, elastic schema change need to add keyword field in order to + // do search + @Getter + private final Function> sortBuilder; + + // We provided a default match query but there are cases where it isn't enough and need more complex + // match, one example is multiple field. Move this logic out of the parser make it easier to read + @Getter + private final Function overridePropertyEqualsToQuery; + + + CQLFeatureFields(String fields, + String displayField, + Function overridePropertyEqualsToQuery, + Function> sortBuilder) { + + this(fields, List.of(displayField), overridePropertyEqualsToQuery, sortBuilder); + } + + CQLFeatureFields(String fields, + List displayField, + Function overridePropertyEqualsToQuery, + Function> sortBuilder) { + + this.searchField = fields; + this.displayField = displayField; + this.overridePropertyEqualsToQuery = overridePropertyEqualsToQuery; + this.sortBuilder = sortBuilder; + } + + @Override + public Query getPropertyEqualToQuery(String literal) { + return null; + } + + @Override + public Query getIntersectsQuery(String literal) { + return null; + } + + @Override + public Query getIsNullQuery() { + return null; + } + + @Override + public Query getLikeQuery(String literal) { + return null; + } + + @Override + public Query getPropertyGreaterThanOrEqualsToQuery(String literal) { + return null; + } + + @Override + public Query getBoundingBoxQuery(TopLeftBottomRightGeoBounds tlbr) { + return null; + } + /** + * Given param, find any of those is not a valid CQLCollectionsField + * @param args - + * @return Invalid enum + */ + public static List findInvalidEnum(List args) { + return args.stream() + .filter(str -> { + try { + CQLFeatureFields.valueOf(str); + return false; + } + catch (IllegalArgumentException e) { + return true; + } + }) + .collect(Collectors.toList()); + } +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureId.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureId.java new file mode 100644 index 00000000..0af82b41 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureId.java @@ -0,0 +1,11 @@ +package au.org.aodn.ogcapi.server.core.model.enumeration; + +public enum FeatureId { + summary("summary"); + + private final String featureId; + + FeatureId(String id) { + this.featureId = id; + } +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureProperty.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureProperty.java index 441cf56e..7c4ba94c 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureProperty.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureProperty.java @@ -8,10 +8,10 @@ public enum FeatureProperty { DEPTH("depth"), COUNT("count"), UUID("uuid"), + GEOMETRY("geometry"), START_TIME("startTime"), END_TIME("endTime"), - COORDINATE_ACCURACY("coordinateAccuracy"), - ; + COORDINATE_ACCURACY("coordinateAccuracy"); private final String value; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/StacBasicField.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/StacBasicField.java index 15072335..f6df6dd2 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/StacBasicField.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/StacBasicField.java @@ -20,7 +20,8 @@ public enum StacBasicField { "organisation_vocabs", "summaries.organisation_vocabs" ), - Links("links", "links") + Links("links", "links"), + Collection("collection", "collection", "collection.keyword"), ; // Field that use to do sort, elastic search treat FieldData (searchField) differently, a searchField is not diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java index 6d1a697c..65d4b3d5 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java @@ -35,7 +35,7 @@ public class CacheNoLandGeometry { @Scheduled(initialDelay = 1000, fixedDelay = Long.MAX_VALUE) @Cacheable("all_noland_geometry") public Map getAllNoLandGeometry() { - ElasticSearchBase.SearchResult result = elasticSearch.searchCollectionBy( + ElasticSearchBase.SearchResult result = elasticSearch.searchCollectionBy( null, null, null, diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 9d374c8f..0b62bd3b 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -1,25 +1,23 @@ package au.org.aodn.ogcapi.server.core.service; -import au.org.aodn.ogcapi.server.core.model.DatasetModel; -import au.org.aodn.ogcapi.server.core.model.DatasetSearchResult; +import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; +import au.org.aodn.ogcapi.server.core.model.StacItemModel; import au.org.aodn.ogcapi.server.core.model.dto.SearchSuggestionsDto; import au.org.aodn.ogcapi.server.core.model.enumeration.*; import au.org.aodn.ogcapi.server.core.parser.elastic.CQLToElasticFilterFactory; import au.org.aodn.ogcapi.server.core.parser.elastic.QueryHandler; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.FieldValue; -import co.elastic.clients.elasticsearch._types.SortOptions; -import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch._types.aggregations.*; import co.elastic.clients.elasticsearch._types.query_dsl.*; import co.elastic.clients.elasticsearch.core.SearchMvtRequest; import co.elastic.clients.elasticsearch.core.SearchRequest; import co.elastic.clients.elasticsearch.core.SearchResponse; import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.core.search_mvt.GridType; +import co.elastic.clients.json.JsonData; import co.elastic.clients.transport.endpoints.BinaryResponse; -import co.elastic.clients.util.ObjectBuilder; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.geotools.filter.text.commons.CompilerUtil; import org.geotools.filter.text.commons.Language; @@ -30,8 +28,9 @@ import org.springframework.http.ResponseEntity; import java.io.IOException; +import java.math.BigDecimal; import java.util.*; -import java.util.function.Supplier; +import java.util.function.BiFunction; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -52,8 +51,8 @@ public class ElasticSearch extends ElasticSearchBase implements Search { @Value("${elasticsearch.vocabs_index.name}") protected String vocabsIndexName; - @Value("${elasticsearch.dataset_index.name}") - protected String datasetIndexName; + @Value("${elasticsearch.cloud_optimized_index.name}") + protected String dataIndexName; @Value("${elasticsearch.search_after.split_regex:\\|\\|}") protected String searchAfterSplitRegex; @@ -147,9 +146,9 @@ this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperatu public ResponseEntity> getAutocompleteSuggestions(String input, String cql, CQLCrsType coor) throws IOException, CQLException { Map> searchSuggestions = new HashMap<>(); - + List> suggestion = this.getSuggestionsByField(input, cql, coor); // extract parameter vocab suggestions - Set parameterVocabSuggestions = this.getSuggestionsByField(input, cql, coor) + Set parameterVocabSuggestions = suggestion .stream() .filter(item -> item.source() != null && item.source().getParameterVocabs() != null && !item.source().getParameterVocabs().isEmpty()) .flatMap(item -> item.source().getParameterVocabs().stream()) @@ -157,7 +156,7 @@ this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperatu .collect(Collectors.toSet()); searchSuggestions.put("suggested_parameter_vocabs", parameterVocabSuggestions); - Set platformVocabSuggestions = this.getSuggestionsByField(input, cql, coor) + Set platformVocabSuggestions = suggestion .stream() .filter(item -> item.source() != null && item.source().getPlatformVocabs() != null && !item.source().getPlatformVocabs().isEmpty()) .flatMap(item -> item.source().getPlatformVocabs().stream()) @@ -165,7 +164,7 @@ this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperatu .collect(Collectors.toSet()); searchSuggestions.put("suggested_platform_vocabs", platformVocabSuggestions); - Set organisationVocabSuggestions = this.getSuggestionsByField(input, cql, coor) + Set organisationVocabSuggestions = suggestion .stream() .filter(item -> item.source() != null && item.source().getOrganisationVocabs() != null && !item.source().getOrganisationVocabs().isEmpty()) .flatMap(item -> item.source().getOrganisationVocabs().stream()) @@ -174,7 +173,7 @@ this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperatu searchSuggestions.put("suggested_organisation_vocabs", organisationVocabSuggestions); // extract abstract phrases suggestions - Set abstractPhrases = this.getSuggestionsByField(input, cql, coor) + Set abstractPhrases = suggestion .stream() .filter(item -> item.source() != null && item.source().getAbstractPhrases() != null && !item.source().getAbstractPhrases().isEmpty()) .flatMap(item -> item.source().getAbstractPhrases().stream()) @@ -185,7 +184,7 @@ this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperatu return new ResponseEntity<>(searchSuggestions, HttpStatus.OK); } - protected ElasticSearchBase.SearchResult searchCollectionsByIds(List ids, Boolean isWithGeometry, String sortBy) { + protected ElasticSearchBase.SearchResult searchCollectionsByIds(List ids, Boolean isWithGeometry, String sortBy) { List queries = new ArrayList<>(); queries.add(MatchQuery.of(m -> m @@ -216,33 +215,33 @@ protected ElasticSearchBase.SearchResult searchCollectionsByIds(List ids filters, null, null, - createSortOptions(sortBy), + createSortOptions(sortBy, CQLFields.class), null, null); } @Override - public ElasticSearchBase.SearchResult searchCollectionWithGeometry(List ids, String sortBy) { + public ElasticSearchBase.SearchResult searchCollectionWithGeometry(List ids, String sortBy) { return searchCollectionsByIds(ids, Boolean.TRUE, sortBy); } @Override - public ElasticSearchBase.SearchResult searchAllCollectionsWithGeometry(String sortBy) { + public ElasticSearchBase.SearchResult searchAllCollectionsWithGeometry(String sortBy) { return searchCollectionsByIds(null, Boolean.TRUE, sortBy); } @Override - public ElasticSearchBase.SearchResult searchCollections(List ids, String sortBy) { + public ElasticSearchBase.SearchResult searchCollections(List ids, String sortBy) { return searchCollectionsByIds(ids, Boolean.FALSE, sortBy); } @Override - public ElasticSearchBase.SearchResult searchAllCollections(String sortBy) { + public ElasticSearchBase.SearchResult searchAllCollections(String sortBy) { return searchCollectionsByIds(null, Boolean.FALSE, sortBy); } @Override - public ElasticSearchBase.SearchResult searchByParameters(List keywords, String cql, List properties, String sortBy, CQLCrsType coor) throws CQLException { + public ElasticSearchBase.SearchResult searchByParameters(List keywords, String cql, List properties, String sortBy, CQLCrsType coor) throws CQLException { if((keywords == null || keywords.isEmpty()) && cql == null) { return searchAllCollections(sortBy); @@ -334,50 +333,12 @@ public ElasticSearchBase.SearchResult searchByParameters(List keywords, filters, properties, searchAfter, - createSortOptions(sortBy), + createSortOptions(sortBy, CQLFields.class), score, maxSize ); } } - /** - * Parse and create a sort option - * ... - * - * @param sortBy - Must be of pattern + | -, + mean asc, - mean desc - * @return List of sort options - */ - protected List createSortOptions(String sortBy) { - if(sortBy == null || sortBy.isEmpty()) return null; - - String[] args = sortBy.split(","); - List sos = new ArrayList<>(); - - for(String arg: args) { - arg = arg.trim(); - if (arg.startsWith("-")) { - CQLFields field = Enum.valueOf(CQLFields.class, arg.substring(1).toLowerCase()); - - if(field.getSortBuilder() != null) { - ObjectBuilder sb = field.getSortBuilder().apply(SortOrder.Desc); - sos.add(sb.build()); - } - } - else { - // Default is +, there is a catch the + will be replaced as space in the property as + means space in url, by taking - // default is ASC we work around the problem, the trim removed the space - CQLFields field = arg.startsWith("+") ? - Enum.valueOf(CQLFields.class, arg.substring(1).toLowerCase()) : - Enum.valueOf(CQLFields.class, arg.toLowerCase()); - - if(field.getSortBuilder() != null) { - ObjectBuilder sb = field.getSortBuilder().apply(SortOrder.Asc); - sos.add(sb.build()); - } - } - } - return sos; - } @Override public BinaryResponse searchCollectionVectorTile(List ids, Integer tileMatrix, Integer tileRow, Integer tileCol) throws IOException { @@ -436,40 +397,213 @@ protected static FieldValue toFieldValue(String s) { // Assume it is string return FieldValue.of(s.trim()); } - + /** + * We will need to create a aggregation for each of the feature query, this one target the summary feature + * which create a summary of the indexed count group by geometry and date range for the cloud optimized data. + * Below code equals this: + * { + * "aggregations": { + * "coordinates": { + * "aggregations": { + * "total_count": { + * "sum": { + * "field": "properties.count" + * } + * }, + * "max_time": { + * "max": { + * "field": "properties.time" + * } + * }, + * "min_time": { + * "min": { + * "field": "properties.time" + * } + * }, + * "coordinates": { + * "top_hits": { + * "size": 1, + * "sort": [ + * { + * "collection.keyword": { + * "order": "asc" + * } + * }, + * { + * "geometry.geometry.coordinates": { + * "order": "asc" + * } + * }, + * ] + * } + * } + * }, + * "composite": { + * "size": 2200, + * "sources": [ + * { + * "collection": { + * "terms": { + * "field": "collection.keyword" + * } + * } + * }, + * { + * "coordinates": { + * "terms": { + * "script": { + * "source": "doc['geometry.geometry.coordinates'].value.toString()", + * "lang": "painless" + * } + * } + * } + * } + * ] + * } + * } + * }, + * "size": 0 + * } + * + * @param collectionId - The metadata set id + * @param properties - The field you want to return + * @param filter - Any filter applied to the summary operation + * @return - Result + */ @Override - public DatasetSearchResult searchDataset( - String collectionId, - String startDate, - String endDate - ) { - List queries = new ArrayList<>(); - queries.add(MatchQuery.of(query -> query - .field("uuid") - .query(collectionId))._toQuery()); + public ElasticSearchBase.SearchResult searchFeatureSummary(String collectionId, List properties, String filter) { + + final String COORDINATES = "coordinates"; + final String TOTAL_COUNT = "total_count"; + final String MIN_TIME = "min_time"; + final String MAX_TIME = "max_time"; + + BiFunction, Map, SearchRequest.Builder> builderSupplier = ( + arguments, afterKey) -> { - Supplier builderSupplier = () -> { SearchRequest.Builder builder = new SearchRequest.Builder(); - builder.index(datasetIndexName) - .size(this.getPageSize()) - .query(query -> query.bool(createBoolQueryForProperties(queries, null, null))); + + builder.query(q -> q + .term(t -> t + .field(CQLFeatureFields.collection.searchField) + .value(arguments.get("collectionId")) + ) + ); + + // Group by lng + CompositeAggregationSource lng = CompositeAggregationSource.of(c -> c.terms(t -> t + .field(CQLFeatureFields.lng.searchField))); + + // Group by lat + CompositeAggregationSource lat = CompositeAggregationSource.of(c -> c.terms(t -> t + .field(CQLFeatureFields.lat.searchField))); + + // Use afterKey to page to another batch of records if exist + Aggregation compose = afterKey == null ? + new Aggregation.Builder().composite(c -> c + .sources(List.of( + Map.of(CQLFeatureFields.lng.name(), lng), + Map.of(CQLFeatureFields.lat.name(), lat)) + ) + .size(pageSize) + ).build() + : + new Aggregation.Builder().composite(c -> c + .sources(List.of( + Map.of(CQLFeatureFields.lng.name(), lng), + Map.of(CQLFeatureFields.lat.name(), lat)) + ) + .size(pageSize) + .after(afterKey) + ).build(); + + + // Sum of count + Aggregation sum = SumAggregation.of(s -> s.field(CQLFeatureFields.count.searchField))._toAggregation(); + + // Min value of field + Aggregation min = MinAggregation.of(s -> s.field(CQLFeatureFields.temporal.searchField))._toAggregation(); + + // Max value of field + Aggregation max = MaxAggregation.of(s -> s.field(CQLFeatureFields.temporal.searchField))._toAggregation(); + + // Field value to return, think of it as select part of SQL + Aggregation field = new Aggregation.Builder().topHits(th -> th.size(1) + .sort(createSortOptions( + String.format("%s,%s", CQLFeatureFields.lng.name(), CQLFeatureFields.lat.name()), + CQLFeatureFields.class))) + .build(); + + Aggregation aggregation = new Aggregation.Builder() + .composite(compose.composite()) + .aggregations(Map.of( + TOTAL_COUNT, sum, + MIN_TIME, min, + MAX_TIME, max, + COORDINATES, field + )) + .build(); + + // There is a limitation that all sort field, assume to be inside the properties + Aggregation nested = new Aggregation.Builder().nested(n -> n + .path("properties") + ) + .aggregations(COORDINATES, aggregation) + .build(); + + + builder.index(dataIndexName) + .size(0) // Do not return hits, only aggregations, that is the hits().hit() section will be empty + .aggregations(COORDINATES, nested); return builder; }; try { - Iterable> response = pagableSearch(builderSupplier, ObjectNode.class, (long) this.getPageSize()); + ElasticSearchBase.SearchResult result = new ElasticSearchBase.SearchResult<>(); + result.setCollections(new ArrayList<>()); + + Map arguments = Map.of( + "collectionId", FieldValue.of(collectionId), + "aggKey", FieldValue.of(COORDINATES) + ); + Iterable response = pageableAggregation(builderSupplier, CompositeBucket.class, arguments, null); - DatasetSearchResult result = new DatasetSearchResult(); + for (CompositeBucket node : response) { + if (node != null) { + StacItemModel. StacItemModelBuilder model = StacItemModel.builder(); - for (var node : response) { - if (node != null && node.source() != null) { - var monthlyData = mapper.readValue(node.source().toPrettyString(), DatasetModel.class); - monthlyData.getData().forEach(result::addDatum); + result.setTotal(result.getTotal() + node.docCount()); + + TopHitsAggregate th = node.aggregations().get(COORDINATES).topHits(); + model.uuid(th.hits().hits().get(0).id()); + + JsonData jd = th.hits().hits().get(0).source(); + if(jd != null) { + Map map = jd.to(Map.class); + BigDecimal lng = BigDecimal.valueOf((double)map.get("lng")); + BigDecimal lat = BigDecimal.valueOf((double)map.get("lat")); + model.geometry(Map.of("geometry", Map.of( + "coordinates", List.of(lng, lat) + ))); + } + + SumAggregate sa = node.aggregations().get(TOTAL_COUNT).sum(); + MinAggregate min = node.aggregations().get(MIN_TIME).min(); + MaxAggregate max = node.aggregations().get(MAX_TIME).max(); + + model.properties(Map.of( + FeatureProperty.COUNT.getValue(), sa.value(), + FeatureProperty.START_TIME.getValue(), min.valueAsString() == null ? "" : min.valueAsString(), + FeatureProperty.END_TIME.getValue(), max.valueAsString() == null ? "" : max.valueAsString() + )); + + result.getCollections().add(model.build()); } } return result; - } catch (Exception e) { + } + catch (Exception e) { log.error("Error while searching dataset.", e); } return null; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java index 571b3699..13824a58 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java @@ -2,9 +2,11 @@ import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFields; +import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFieldsInterface; import au.org.aodn.ogcapi.server.core.model.enumeration.StacBasicField; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.*; +import co.elastic.clients.elasticsearch._types.aggregations.*; import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; import co.elastic.clients.elasticsearch._types.query_dsl.Query; import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders; @@ -14,6 +16,7 @@ import co.elastic.clients.elasticsearch.core.SearchResponse; import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.core.search.SourceConfig; +import co.elastic.clients.util.ObjectBuilder; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -26,6 +29,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; import java.util.function.Supplier; /** @@ -46,12 +50,49 @@ abstract class ElasticSearchBase { @Getter @Setter - public static class SearchResult { - Long total; + public static class SearchResult { + Long total = 0L; List sortValues; - List collections; + List collections; } + /** + * Parse and create a sort option + * ... + * + * @param sortBy - Must be of pattern + | -, + mean asc, - mean desc + * @return List of sort options + */ + protected & CQLFieldsInterface> List createSortOptions(String sortBy, Class enumClass) { + if(sortBy == null || sortBy.isEmpty()) return null; + String[] args = sortBy.split(","); + List sos = new ArrayList<>(); + + for(String arg: args) { + arg = arg.trim(); + if (arg.startsWith("-")) { + CQLFields field = Enum.valueOf(CQLFields.class, arg.substring(1).toLowerCase()); + + if(field.getSortBuilder() != null) { + ObjectBuilder sb = field.getSortBuilder().apply(SortOrder.Desc); + sos.add(sb.build()); + } + } + else { + // Default is +, there is a catch the + will be replaced as space in the property as + means space in url, by taking + // default is ASC we work around the problem, the trim removed the space + E field = arg.startsWith("+") ? + Enum.valueOf(enumClass, arg.substring(1).toLowerCase()) : + Enum.valueOf(enumClass, arg.toLowerCase()); + + if(field.getSortBuilder() != null) { + ObjectBuilder sb = field.getSortBuilder().apply(SortOrder.Asc); + sos.add(sb.build()); + } + } + } + return sos; + } /** * Construct the skeleton of in the elastic query and fill in values * @param must - The must portion of Elastic query @@ -107,7 +148,7 @@ protected SearchRequest buildSearchAsYouTypeRequest( * @param properties - The fields you want to return in the search, you can search a field but not include in the return * @return - The search result from Elastic query and format in StacCollectionModel */ - protected SearchResult searchCollectionBy(final List queries, + protected SearchResult searchCollectionBy(final List queries, final List should, final List filters, final List properties, @@ -189,9 +230,9 @@ protected SearchResult searchCollectionBy(final List queries, try { log.info("Start search {} {}", ZonedDateTime.now(), Thread.currentThread().getName()); - Iterable> response = pagableSearch(builderSupplier, ObjectNode.class, maxSize); + Iterable> response = pageableSearch(builderSupplier, ObjectNode.class, maxSize); - SearchResult result = new SearchResult(); + SearchResult result = new SearchResult<>(); result.collections = new ArrayList<>(); result.total = countRecordsHit(builderSupplier); @@ -272,10 +313,10 @@ protected Long countRecordsHit(Supplier requestBuilder) { * @return - The items that matches the query mentioned in the requestBuilder * @param A generic type for Elastic query */ - protected Iterable> pagableSearch(Supplier requestBuilder, Class clazz, Long maxSize) { + protected Iterable> pageableSearch(Supplier requestBuilder, Class clazz, Long maxSize) { try { SearchRequest sr = requestBuilder.get().build(); - log.debug("Final elastic search payload {}", sr.toString()); + log.debug("Final elastic search payload {}", sr); final AtomicLong count = new AtomicLong(0); final AtomicReference> response = new AtomicReference<>( @@ -337,6 +378,111 @@ public Hit next() { } return Collections.emptySet(); } + /** + * There is a limit of how many record a query can return, this mean the record return may not be full set, you + * need to keep loading until you reach the end of records + * + * @param requestBuilder, assume it is sorted with order, what order isn't important, as long as it is sorted + * @param clazz - The type + * @return - The items that matches the query mentioned in the requestBuilder + * @param A generic type for Elastic query + */ + protected Iterable pageableAggregation( + BiFunction, Map, SearchRequest.Builder> requestBuilder, + Class clazz, + Map arguments, + Long maxSize) { + try { + SearchRequest sr = requestBuilder.apply(arguments, null).build(); + log.debug("Final elastic aggregation payload {}", sr); + + final AtomicLong count = new AtomicLong(0); + final AtomicReference> response = new AtomicReference<>( + esClient.search(sr, clazz) + ); + + return () -> new Iterator<>() { + private int index = 0; + + @Override + public boolean hasNext() { + // No need continue if we already hit the end + if(maxSize != null) { + return count.get() < maxSize; + } + + Aggregate ags = response.get() + .aggregations() + .get(arguments.get("aggKey").stringValue()) + .nested() + .aggregations() + .get(arguments.get("aggKey").stringValue()); + + Buckets stb = ags.composite().buckets(); + + // If we hit the end, that means we have iterated to end of page. + if (index < stb.array().size()) { + return true; + } + else { + // If last index is zero that mean nothing found already, so no need to look more + if (index == 0) return false; + + // Load next batch + try { + // Get the last sorted value from the last batch + // Use the last builder and append the searchAfter values + SearchRequest request = requestBuilder.apply(arguments, ags.composite().afterKey()).build(); + log.debug("Final elastic aggregation payload {}", request.toString()); + + response.set(esClient.search(request, clazz)); + // Reset counter from start + index = 0; + + ags = response.get() + .aggregations() + .get(arguments.get("aggKey").stringValue()) + .nested() + .aggregations() + .get(arguments.get("aggKey").stringValue()); + + stb = ags.composite().buckets(); + + return index < stb.array().size(); + } + catch(IOException ieo) { + throw new RuntimeException(ieo); + } + } + } + + @Override + public T next() { + count.incrementAndGet(); + + Aggregate ags = response.get() + .aggregations() + .get(arguments.get("aggKey").stringValue()) + .nested() + .aggregations() + .get(arguments.get("aggKey").stringValue()); + + Buckets stb = ags.composite().buckets(); + + if(index < stb.array().size()) { + return clazz.cast(stb.array().get(index++)); + } + else { + return null; + } + } + }; + } + catch(Exception e) { + log.error("Fail to fetch record", e); + } + return Collections.emptySet(); + } protected StacCollectionModel formatResult(ObjectNode nodes) { try { diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java index aad2a437..3841423a 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java @@ -1,6 +1,9 @@ package au.org.aodn.ogcapi.server.core.service; +import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; +import au.org.aodn.ogcapi.server.core.model.StacItemModel; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType; +import au.org.aodn.ogcapi.server.core.model.enumeration.FeatureId; import au.org.aodn.ogcapi.server.core.model.enumeration.OGCMediaTypeMapper; import au.org.aodn.ogcapi.server.core.exception.CustomException; import au.org.aodn.ogcapi.server.core.parser.stac.CQLToStacFilterFactory; @@ -14,6 +17,7 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import java.math.BigDecimal; import java.util.List; import java.util.function.BiFunction; @@ -28,22 +32,41 @@ public abstract class OGCApiService { protected Search search; /** - * You can find conformance id here https://docs.ogc.org/is/19-072/19-072.html#ats_core + * You can find conformance id + * here * @return List of string contains conformance */ public abstract List getConformanceDeclaration(); + public ResponseEntity getFeature(String collectionId, + FeatureId fid, + List properties, + String filter, + BiFunction, Filter, R> converter) throws Exception { + switch(fid) { + case summary -> { + ElasticSearch.SearchResult result = search.searchFeatureSummary(collectionId, properties, filter); + return ResponseEntity.ok() + .body(converter.apply(result, null)); + } + default -> { + // Individual item + return new ResponseEntity<>(HttpStatus.NOT_IMPLEMENTED); + } + } + } + public ResponseEntity getCollectionList(List keywords, String filter, List properties, String sortBy, OGCMediaTypeMapper f, CQLCrsType coor, - BiFunction converter) { + BiFunction, Filter, R> converter) { try { switch (f) { case json -> { - ElasticSearchBase.SearchResult result = search.searchByParameters(keywords, filter, properties, sortBy, coor); + ElasticSearchBase.SearchResult result = search.searchByParameters(keywords, filter, properties, sortBy, coor); CQLToStacFilterFactory factory = CQLToStacFilterFactory.builder() .cqlCrsType(coor) @@ -93,7 +116,7 @@ public ResponseEntity getCollectionList(List keywords, * @param filter - Any existing filter * @return - A combined filter with datetime rewrite. */ - public static String processDatetimeParameter(String datetime, String filter) { + public static String processDatetimeParameter(String fieldName, String datetime, String filter) { // TODO: How to handle this? e.g how to know if it is before or after if ?datetime= @@ -117,7 +140,7 @@ else if (datetime.contains("/") && !datetime.contains("..")) { } if(d != null) { - f = String.format("temporal %s %s", operator, d); + f = String.format("%s %s %s", fieldName, operator, d); } if((filter == null || filter.isEmpty())) { @@ -132,4 +155,28 @@ else if (datetime.contains("/") && !datetime.contains("..")) { } } } + /** + * Convert the bbox parameter to CQL + * @param bbox + * @param filter + * @return + */ + public static String processBBoxParameter(String fieldName, List bbox, String filter) { + String f = null; + if(bbox.size() == 4) { + // 2D + f = String.format("BBOX(%s,%s,%s,%s,%s)", fieldName, bbox.get(0), bbox.get(1), bbox.get(2), bbox.get(3)); + } + else if(bbox.size() == 6) { + // 3D + f = String.format("BBOX(%s,%s,%s,%s,%s,%s,%s)", fieldName, bbox.get(0), bbox.get(1), bbox.get(2), bbox.get(3), bbox.get(4), bbox.get(5)); + } + + if(f == null) { + return filter; + } + else { + return String.join(" AND ", filter, f); + } + } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/Search.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/Search.java index dcde8713..256b6ca0 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/Search.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/Search.java @@ -1,6 +1,7 @@ package au.org.aodn.ogcapi.server.core.service; -import au.org.aodn.ogcapi.server.core.model.DatasetSearchResult; +import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; +import au.org.aodn.ogcapi.server.core.model.StacItemModel; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType; import co.elastic.clients.transport.endpoints.BinaryResponse; import org.springframework.http.ResponseEntity; @@ -10,14 +11,14 @@ import java.util.Map; public interface Search { - ElasticSearchBase.SearchResult searchCollectionWithGeometry(List ids, String sortBy) throws Exception; - ElasticSearchBase.SearchResult searchAllCollectionsWithGeometry(String sortBy) throws Exception; + ElasticSearchBase.SearchResult searchCollectionWithGeometry(List ids, String sortBy) throws Exception; + ElasticSearchBase.SearchResult searchAllCollectionsWithGeometry(String sortBy) throws Exception; - ElasticSearchBase.SearchResult searchCollections(List ids, String sortBy); - ElasticSearchBase.SearchResult searchAllCollections(String sortBy) throws Exception; - DatasetSearchResult searchDataset(String collectionId, String startDate, String endDate) throws Exception; + ElasticSearchBase.SearchResult searchCollections(List ids, String sortBy); + ElasticSearchBase.SearchResult searchAllCollections(String sortBy) throws Exception; + ElasticSearchBase.SearchResult searchFeatureSummary(String collectionId, List properties, String filter) throws Exception; - ElasticSearchBase.SearchResult searchByParameters( + ElasticSearchBase.SearchResult searchByParameters( List targets, String filter, List properties, diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/CommonUtils.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/CommonUtils.java new file mode 100644 index 00000000..1e7a1e45 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/CommonUtils.java @@ -0,0 +1,17 @@ +package au.org.aodn.ogcapi.server.core.util; + +import java.util.Optional; +import java.util.function.Supplier; + +public class CommonUtils { + public static Optional safeGet(Supplier supplier) { + try { + return Optional.of(supplier.get()); + } catch ( + NullPointerException + | IndexOutOfBoundsException + | ClassCastException ignored) { + return Optional.empty(); + } + } +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java index 64d01096..f0c26118 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java @@ -3,9 +3,22 @@ import au.org.aodn.ogcapi.features.api.CollectionsApi; import au.org.aodn.ogcapi.features.model.Collection; import au.org.aodn.ogcapi.features.model.Collections; +import au.org.aodn.ogcapi.features.model.Exception; import au.org.aodn.ogcapi.features.model.FeatureCollectionGeoJSON; import au.org.aodn.ogcapi.features.model.FeatureGeoJSON; +import au.org.aodn.ogcapi.server.core.mapper.StacToFeatureCollection; +import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFields; +import au.org.aodn.ogcapi.server.core.model.enumeration.FeatureId; +import au.org.aodn.ogcapi.server.core.service.OGCApiService; import io.swagger.v3.oas.annotations.Hidden; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import jakarta.validation.Valid; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -21,47 +34,104 @@ public class RestApi implements CollectionsApi { @Autowired protected RestServices featuresService; + @Autowired + protected StacToFeatureCollection stacToFeatureCollection; + @Override public ResponseEntity describeCollection(String collectionId) { return featuresService.getCollection(collectionId, null); } + @Hidden @Override public ResponseEntity getFeature(String collectionId, String featureId) { return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build(); } + @Operation( + summary = "fetch a single feature", + description = "Fetch the feature with id `featureId` in the feature collection with id `collectionId`. Use content negotiation to request HTML or GeoJSON.", + tags = {"Data"} + ) + @ApiResponses({@ApiResponse( + responseCode = "200", + description = "fetch the feature with id `featureId` in the feature collection with id `collectionId`", + content = {@Content( + mediaType = "application/geo+json", + schema = @Schema( + implementation = FeatureGeoJSON.class + ) + )} + ), @ApiResponse( + responseCode = "404", + description = "The requested resource does not exist on the server. For example, a path parameter had an incorrect value." + ), @ApiResponse( + responseCode = "500", + description = "A server error occurred.", + content = {@Content( + mediaType = "application/json", + schema = @Schema( + implementation = Exception.class + ) + )} + )}) @RequestMapping( - value = {"/collections/{collectionId}/items"}, + value = {"/collections/{collectionId}/items/{featureId}"}, produces = {"application/geo+json", "text/html", "application/json"}, method = {RequestMethod.GET} ) - public ResponseEntity getFeatures( - + ResponseEntity getFeature( + @Parameter(in = ParameterIn.PATH,description = "local identifier of a collection",required = true,schema = @Schema) @PathVariable("collectionId") String collectionId, - @RequestParam(value = "start_datetime", required = false) String startDate, - @RequestParam(value = "end_datetime", required = false) String endDate, - // keep these two parameters for future usage - @RequestParam(value= "zoom", required = false) Double zoomLevel, - @RequestParam(value="bbox", required = false) List bbox - ) { - return featuresService.getSummarizedDataset(collectionId, startDate, endDate); - } + @Parameter(in = ParameterIn.PATH,description = "local identifier of a feature",required = true,schema = @Schema) + @PathVariable("featureId") String featureId, + @Parameter(in = ParameterIn.QUERY, description = "Property to be return" ,schema=@Schema()) + @Valid @RequestParam(value = "properties", required = false) List properties, + @Parameter(in = ParameterIn.QUERY, description = "Only records that have a geometry that intersects the bounding box are selected. The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or depth): * Lower left corner, coordinate axis 1 * Lower left corner, coordinate axis 2 * Minimum value, coordinate axis 3 (optional) * Upper right corner, coordinate axis 1 * Upper right corner, coordinate axis 2 * Maximum value, coordinate axis 3 (optional) The coordinate reference system of the values is WGS 84 long/lat (http://www.opengis.net/def/crs/OGC/1.3/CRS84) unless a different coordinate reference system is specified in the parameter `bbox-crs`. For WGS 84 longitude/latitude the values are in most cases the sequence of minimum longitude, minimum latitude, maximum longitude and maximum latitude. However, in cases where the box spans the antimeridian the first value (west-most box edge) is larger than the third value (east-most box edge). If the vertical axis is included, the third and the sixth number are the bottom and the top of the 3-dimensional bounding box. If a record has multiple spatial geometry properties, it is the decision of the server whether only a single spatial geometry property is used to determine the extent or all relevant geometries." ,schema=@Schema()) + @Valid @RequestParam(value = "bbox", required = false) List bbox, + @Parameter(in = ParameterIn.QUERY, description = "Either a date-time or an interval, open or closed. Date and time expressions adhere to RFC 3339. Open intervals are expressed using double-dots. Examples: * A date-time: \"2018-02-12T23:20:50Z\" * A closed interval: \"2018-02-12T00:00:00Z/2018-03-18T12:31:12Z\" * Open intervals: \"2018-02-12T00:00:00Z/..\" or \"../2018-03-18T12:31:12Z\" Only records that have a temporal property that intersects the value of `datetime` are selected. It is left to the decision of the server whether only a single temporal property is used to determine the extent or all relevant temporal properties." ,schema=@Schema()) + @Valid @RequestParam(value = "datetime", required = false) String datetime) { + + String filter = null; + if (datetime != null) { + filter = OGCApiService.processDatetimeParameter(CQLFields.temporal.name(), datetime, filter); + } + if (bbox != null) { + filter = OGCApiService.processBBoxParameter(CQLFields.geometry.name(), bbox, filter); + } + try { + FeatureId fid = FeatureId.valueOf(FeatureId.class, featureId); + return featuresService.getFeature( + collectionId, + fid, + properties, + filter != null ? "filter=" + filter : null, + stacToFeatureCollection::convert + ); + } + catch(java.lang.Exception e) { + return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build(); + } + } /** - * Hidden because we want to have a more functional implementation + * + * @param collectionId - The collection id + * @param limit - Limit of result return + * @param bbox - Bounding box that bounds the result set. In case of multiple bounding box, you need to issue multiple query + * @param datetime - Start/end time + * @return - The data that matches the filter criteria */ - @Hidden @Override public ResponseEntity getFeatures(String collectionId, Integer limit, List bbox, String datetime) { - return null; + return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build(); } /** - * @Hidden effectively disable this REST point because it is common in many places and + * Hidden effectively disable this REST point because it is common in many places and * should not implement it here, @Hidden disable swagger doc too - * @return + * @return - Not implemented */ @Hidden @Override diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java index 8fa14992..f0e68ab1 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java @@ -1,8 +1,8 @@ package au.org.aodn.ogcapi.server.features; import au.org.aodn.ogcapi.features.model.Collection; -import au.org.aodn.ogcapi.features.model.FeatureCollectionGeoJSON; import au.org.aodn.ogcapi.server.core.mapper.StacToCollection; +import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; import au.org.aodn.ogcapi.server.core.service.ElasticSearch; import au.org.aodn.ogcapi.server.core.service.OGCApiService; import lombok.extern.slf4j.Slf4j; @@ -13,8 +13,8 @@ import java.util.List; import java.util.NoSuchElementException; -@Service("FeaturesRestService") @Slf4j +@Service("FeaturesRestService") public class RestServices extends OGCApiService { @Autowired @@ -26,7 +26,7 @@ public List getConformanceDeclaration() { } public ResponseEntity getCollection(String id, String sortBy) throws NoSuchElementException { - ElasticSearch.SearchResult model = search.searchCollections(List.of(id), sortBy); + ElasticSearch.SearchResult model = search.searchCollections(List.of(id), sortBy); if (!model.getCollections().isEmpty()) { if(model.getCollections().size() > 1) { @@ -40,19 +40,4 @@ public ResponseEntity getCollection(String id, String sortBy) throws return ResponseEntity.notFound().build(); } } - - public ResponseEntity getSummarizedDataset( - String collectionId, - String startDate, - String endDate - ) { - try { - var result = search.searchDataset(collectionId, startDate, endDate); - return ResponseEntity.ok() - .body(result.getSummarizedDataset()); - } catch (Exception e) { - log.error("Error while getting dataset", e); - return ResponseEntity.internalServerError().build(); - } - } } diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml index 96305610..7c866e21 100644 --- a/server/src/main/resources/application.yaml +++ b/server/src/main/resources/application.yaml @@ -10,8 +10,8 @@ elasticsearch: name: dev_portal_records vocabs_index: name: vocabs_index - dataset_index: - name: dataset_index + cloud_optimized_index: + name: es-coindexer-dev serverUrl: http://localhost:9200 apiKey: search_as_you_type: diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java b/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java index 6c36dcea..f76d0f59 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/BaseTestClass.java @@ -11,6 +11,8 @@ import co.elastic.clients.transport.rest_client.RestClientTransport; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.annotation.PostConstruct; +import lombok.extern.slf4j.Slf4j; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; @@ -33,6 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +@Slf4j public class BaseTestClass { @LocalServerPort @@ -53,9 +56,15 @@ public class BaseTestClass { @Value("${elasticsearch.index.name}") protected String record_index_name; + @Value("${elasticsearch.cloud_optimized_index.name}") + protected String co_data_index_name; + @Value("${elasticsearch.vocabs_index.name}") protected String vocabs_index_name; + @Value("${elasticsearch.cloud_optimized_index.name}") + protected String data_index_name; + protected Logger logger = LoggerFactory.getLogger(BaseTestClass.class); protected String getBasePath() { @@ -66,12 +75,18 @@ protected String getExternalBasePath() { return "http://localhost:" + port + "/api/v1/ogc/ext"; } - protected void clearElasticIndex() throws IOException { + protected List> schemas; - List> schemas = List.of( + @PostConstruct + public void initSchemas() { + schemas = List.of( + Map.of("name", record_index_name, "mapping", "portal_records_index_schema.json"), Map.of("name", vocabs_index_name, "mapping", "vocabs_index_schema.json"), - Map.of("name", record_index_name, "mapping", "portal_records_index_schema.json") + Map.of("name", data_index_name, "mapping", "data_index_schema.json") ); + } + + protected void clearElasticIndex() { logger.debug("Clear elastic index"); try { @@ -99,11 +114,6 @@ protected void clearElasticIndex() throws IOException { protected void createElasticIndex() { - List> schemas = List.of( - Map.of("name", record_index_name, "mapping", "portal_records_index_schema.json"), - Map.of("name", vocabs_index_name, "mapping", "vocabs_index_schema.json") - ); - schemas.forEach(schema -> { try { // TODO: This file should come from indexer jar when CodeArtifact in place @@ -119,7 +129,7 @@ protected void createElasticIndex() { // Ignore it, happens when index already created } } catch (IOException e) { - e.printStackTrace(); + log.error("Error", e); } }); } @@ -201,14 +211,14 @@ protected void bulkIndexVocabs(List vocabs) throws IOException { } } - protected void insertJsonToElasticIndex(String... filenames) throws IOException { + protected void insertJsonToElasticIndex(String index, String[] filenames) throws IOException { // Now insert json to index for(String filename : filenames) { File j = ResourceUtils.getFile("classpath:databag/" + filename); try(Reader reader = new FileReader(j)) { IndexResponse indexResponse = client.index(i -> i - .index(record_index_name) + .index(index) .withJson(reader)); logger.info("Sample file {}, indexed with response : {}", filename, indexResponse); @@ -221,7 +231,7 @@ protected void insertJsonToElasticIndex(String... filenames) throws IOException // Check the number of doc store inside the ES instance is correct SearchRequest.Builder b = new SearchRequest.Builder() - .index(record_index_name) + .index(index) .query(QueryBuilders.matchAll().build()._toQuery()); SearchRequest request = b.build(); @@ -233,6 +243,14 @@ protected void insertJsonToElasticIndex(String... filenames) throws IOException assertEquals(filenames.length, response.hits().hits().size(), "Number of docs stored is correct"); } + protected void insertJsonToElasticRecordIndex(String... filenames) throws IOException { + this.insertJsonToElasticIndex(record_index_name, filenames); + } + + protected void insertJsonToElasticCODataIndex(String... filenames) throws IOException { + this.insertJsonToElasticIndex(co_data_index_name, filenames); + } + protected Response getClusterHealth() throws IOException { return transport.restClient().performRequest(new Request("GET", "/_cluster/health")); } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java index fb2eb088..cfdaeec6 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java @@ -30,12 +30,12 @@ public void beforeClass() { } @AfterAll - public void clear() throws IOException { + public void clear() { super.clearElasticIndex(); } @BeforeEach - public void afterTest() throws IOException { + public void afterTest() { super.clearElasticIndex(); } @@ -44,13 +44,13 @@ public void verifyApiGet() { RestApi api = new RestApi(); ResponseEntity response = api.apiGet(OGCMediaTypeMapper.json.toString()); - assertEquals(response.getStatusCode(), HttpStatus.TEMPORARY_REDIRECT, "Incorrect redirect"); - assertEquals(Objects.requireNonNull(response.getHeaders().getLocation()).getPath() , "/api/v1/ogc/api-docs/v3", "Incorrect path"); + assertEquals(HttpStatus.TEMPORARY_REDIRECT, response.getStatusCode(), "Incorrect redirect"); + assertEquals("/api/v1/ogc/api-docs/v3", Objects.requireNonNull(response.getHeaders().getLocation()).getPath(), "Incorrect path"); response = api.apiGet(OGCMediaTypeMapper.html.toString()); - assertEquals(response.getStatusCode(), HttpStatus.TEMPORARY_REDIRECT, "Incorrect redirect"); - assertEquals(Objects.requireNonNull(response.getHeaders().getLocation()).getPath(), "/api/v1/ogc/swagger-ui/index.html", "Incorrect path"); + assertEquals(HttpStatus.TEMPORARY_REDIRECT, response.getStatusCode(), "Incorrect redirect"); + assertEquals("/api/v1/ogc/swagger-ui/index.html", Objects.requireNonNull(response.getHeaders().getLocation()).getPath(), "Incorrect path"); } @Test @@ -65,7 +65,7 @@ public void verifyClusterIsHealthy() throws IOException { */ @Test public void verifyApiCollectionsQueryOnText1() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" ); @@ -109,7 +109,7 @@ public void verifyApiCollectionsQueryOnText1() throws IOException { */ @Test public void verifyApiCollectionsQueryOnText2() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "073fde5a-bff3-1c1f-e053-08114f8c5588.json", "9fdb1eee-bc28-43a9-88c5-972324784837.json" @@ -133,7 +133,7 @@ public void verifyApiCollectionsQueryOnText2() throws IOException { */ @Test public void verifyDateTimeAfterBounds() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "5c418118-2581-4936-b6fd-d6bedfe74f62.json" @@ -171,7 +171,7 @@ public void verifyDateTimeAfterBounds() throws IOException { */ @Test public void verifyDateTimeBeforeBounds() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "5c418118-2581-4936-b6fd-d6bedfe74f62.json" @@ -197,7 +197,7 @@ public void verifyDateTimeBeforeBounds() throws IOException { */ @Test public void verifyDateTimeBetweenBounds() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "5c418118-2581-4936-b6fd-d6bedfe74f62.json" @@ -231,7 +231,7 @@ public void verifyDateTimeBetweenBounds() throws IOException { */ @Test public void verifyDateTimeBoundsWithDiscreteTime() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "caf7220a-19e0-4a7f-9af6-eade6c79a47a.json" // This one have two start/end @@ -272,7 +272,7 @@ public void verifyDateTimeBoundsWithDiscreteTime() throws IOException { */ @Test public void verifyPropertiesParameter() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json" ); @@ -292,7 +292,7 @@ public void verifyPropertiesParameter() throws IOException { */ @Test public void verifyCQLPropertyIsNullIsNotNull() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", // Provider null "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" // Provider not null ); @@ -317,7 +317,7 @@ public void verifyCQLPropertyIsNullIsNotNull() throws IOException { */ @Test public void verifyCQLPropertyEqualsOperation() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", // Provider null "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" // Provider is IMOS ); @@ -336,7 +336,7 @@ public void verifyCQLPropertyEqualsOperation() throws IOException { */ @Test public void verifyCQLPropertyAndOperation() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", // Provider null "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" // Provider is IMOS ); @@ -363,7 +363,7 @@ public void verifyCQLPropertyAndOperation() throws IOException { */ @Test public void verifyParameterTextSearchMatch() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", // Provider null "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" // Provider is IMOS ); @@ -380,7 +380,7 @@ public void verifyParameterTextSearchMatch() throws IOException { @Test public void verifyParameterParameterVocabsSearchMatch() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" @@ -408,7 +408,7 @@ public void verifyParameterParameterVocabsSearchMatch() throws IOException { */ @Test public void verifyCQLPropertyOrOperation() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", // Provider null "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" // Provider is IMOS ); @@ -437,7 +437,7 @@ public void verifyCQLPropertyOrOperation() throws IOException { */ @Test public void verifyCQLPropertyIntersectOperation() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "b299cdcd-3dee-48aa-abdd-e0fcdbb9cadc.json" ); // Intersect with this polygon @@ -465,7 +465,7 @@ public void verifyCQLPropertyIntersectOperation() throws IOException { */ @Test public void verifyCQLPropertyDatasetGroup() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "5c418118-2581-4936-b6fd-d6bedfe74f62.json", // Provider null "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" // Provider is IMOS ); @@ -484,7 +484,7 @@ public void verifyCQLPropertyDatasetGroup() throws IOException { */ @Test public void verifyCQLPropertyScore() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" @@ -495,8 +495,8 @@ public void verifyCQLPropertyScore() throws IOException { // Make sure OR not work as it didn't make sense to use or with setting ResponseEntity error = testRestTemplate.getForEntity(getBasePath() + "/collections?filter=score>=2 OR parameter_vocabs='wave'", ErrorResponse.class); - assertEquals(error.getStatusCode(), HttpStatus.INTERNAL_SERVER_ERROR); - assertEquals(Objects.requireNonNull(error.getBody()).getMessage(), "Or combine with query setting do not make sense", "correct error"); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, error.getStatusCode()); + assertEquals("Or combine with query setting do not make sense", Objects.requireNonNull(error.getBody()).getMessage(), "correct error"); // Lower score but the fuzzy is now with operator AND, therefore it will try to match all words 'dataset' and 'includes' with fuzzy collections = testRestTemplate.getForEntity(getBasePath() + "/collections?q='dataset includes'&filter=score>=1", Collections.class); @@ -514,7 +514,7 @@ public void verifyCQLPropertyScore() throws IOException { */ @Test public void verifyCQLFuzzyKey() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" @@ -548,7 +548,7 @@ public void verifyErrorMessageCreated() { */ @Test public void verifySortBy() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" @@ -585,7 +585,7 @@ public void verifySortBy() throws IOException { */ @Test public void verifySortByTemporalCorrect() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "073fde5a-bff3-1c1f-e053-08114f8c5588.json", "5c418118-2581-4936-b6fd-d6bedfe74f62.json", "19da2ce7-138f-4427-89de-a50c724f5f54.json", diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestExtApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestExtApiTest.java index 8061cf98..a391cc05 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestExtApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestExtApiTest.java @@ -24,17 +24,17 @@ public class RestExtApiTest extends BaseTestClass { @BeforeAll - public void beforeClass() throws IOException { + public void beforeClass() { super.createElasticIndex(); } @AfterAll - public void clear() throws IOException { + public void clear() { super.clearElasticIndex(); } @BeforeEach - public void afterTest() throws IOException { + public void afterTest() { super.clearElasticIndex(); } @@ -48,11 +48,11 @@ public void verifyClusterIsHealthy() throws IOException { /** * - * @throws IOException + * @throws IOException - not expect to throw */ @Test public void verifyApiResponseOnIncompleteInput() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" ); @@ -69,11 +69,11 @@ public void verifyApiResponseOnIncompleteInput() throws IOException { /** * - * @throws IOException + * @throws IOException - not expect to throw */ @Test public void verifyApiResponseOnCompleteInput() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" ); @@ -89,11 +89,11 @@ public void verifyApiResponseOnCompleteInput() throws IOException { /** * - * @throws IOException + * @throws IOException - not expect to throw */ @Test public void verifyApiResponseOnTypoInputNoParameterVocabsFilter() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" @@ -110,7 +110,7 @@ public void verifyApiResponseOnTypoInputNoParameterVocabsFilter() throws IOExcep @Test public void verifyApiResponseOnTypoInputSingleParameterVocabsFilter() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" @@ -129,7 +129,7 @@ public void verifyApiResponseOnTypoInputSingleParameterVocabsFilter() throws IOE @Test public void verifyApiResponseOnTypoInputMultipleParameterVocabsFilter() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" @@ -148,7 +148,7 @@ public void verifyApiResponseOnTypoInputMultipleParameterVocabsFilter() throws I @Test public void verifyApiResponseOnTypoInputMultipleParameterVocabsFilterNoResults() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json" @@ -165,7 +165,7 @@ public void verifyApiResponseOnTypoInputMultipleParameterVocabsFilterNoResults() public void verifyApiResponseOnParameterVocabSuggestions() throws IOException { super.insertTestVocabs(); - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "19da2ce7-138f-4427-89de-a50c724f5f54.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" ); @@ -192,7 +192,7 @@ public void verifyApiResponseOnParameterVocabSuggestions() throws IOException { public void verifyApiResponseOnPlatformVocabSuggestions() throws IOException { super.insertTestVocabs(); - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "record_with_parameter_platform_organisation_vocabs.json" ); @@ -238,7 +238,7 @@ public void verifyApiResponseOnPlatformVocabSuggestions() throws IOException { public void verifyApiResponseOnOrganisationVocabSuggestions() throws IOException { super.insertTestVocabs(); - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "record_with_parameter_platform_organisation_vocabs.json" ); diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/OGCApiServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/OGCApiServiceTest.java index d36c68c3..162e6584 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/OGCApiServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/OGCApiServiceTest.java @@ -1,5 +1,6 @@ package au.org.aodn.ogcapi.server.core.service; +import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFields; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -12,22 +13,22 @@ public class OGCApiServiceTest { @Test public void verifyProcessDatetimeParameter() { - String o = OGCApiService.processDatetimeParameter("../2021-10-10", ""); + String o = OGCApiService.processDatetimeParameter(CQLFields.temporal.name(), "../2021-10-10", ""); assertEquals( "temporal before 2021-10-10", o, "Before incorrect1"); - o = OGCApiService.processDatetimeParameter("/2021-10-10", ""); + o = OGCApiService.processDatetimeParameter(CQLFields.temporal.name(), "/2021-10-10", ""); assertEquals( "temporal before 2021-10-10", o, "Before incorrect2"); - o = OGCApiService.processDatetimeParameter("2021-10-10/", ""); + o = OGCApiService.processDatetimeParameter(CQLFields.temporal.name(),"2021-10-10/", ""); assertEquals( "temporal after 2021-10-10", o, "After incorrect1"); - o = OGCApiService.processDatetimeParameter("2021-10-10/..", ""); + o = OGCApiService.processDatetimeParameter(CQLFields.temporal.name(),"2021-10-10/..", ""); assertEquals( "temporal after 2021-10-10", o, "After incorrect1"); - o = OGCApiService.processDatetimeParameter("2021-10-10/2022-10-10", ""); + o = OGCApiService.processDatetimeParameter(CQLFields.temporal.name(),"2021-10-10/2022-10-10", ""); assertEquals( "temporal during 2021-10-10/2022-10-10", o, "During incorrect1"); - o = OGCApiService.processDatetimeParameter("/2021-10-10", "geometry is null"); + o = OGCApiService.processDatetimeParameter(CQLFields.temporal.name(),"/2021-10-10", "geometry is null"); assertEquals( "geometry is null AND temporal before 2021-10-10", o, "Before plus filter incorrect1"); } } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 47fdbda9..0af66a61 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -1,8 +1,12 @@ package au.org.aodn.ogcapi.server.features; import au.org.aodn.ogcapi.features.model.Collection; +import au.org.aodn.ogcapi.features.model.FeatureCollectionGeoJSON; +import au.org.aodn.ogcapi.features.model.FeatureGeoJSON; +import au.org.aodn.ogcapi.features.model.PointGeoJSON; import au.org.aodn.ogcapi.server.BaseTestClass; import au.org.aodn.ogcapi.server.core.model.ExtendedCollections; +import au.org.aodn.ogcapi.server.core.model.enumeration.FeatureProperty; import org.junit.jupiter.api.*; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; @@ -33,12 +37,12 @@ public void beforeClass() { } @AfterAll - public void clear() throws IOException { + public void clear() { super.clearElasticIndex(); } @BeforeEach - public void afterTest() throws IOException { + public void afterTest() { super.clearElasticIndex(); } @@ -57,7 +61,7 @@ public void verifyCorrectInternalPagingLargeData() throws IOException { // Given 6 records and we set page to 4, that means each query elastic return 4 record only // and the logic to load the reset can kick in. - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "5c418118-2581-4936-b6fd-d6bedfe74f62.json", "19da2ce7-138f-4427-89de-a50c724f5f54.json", "516811d7-cd1e-207a-e0440003ba8c79dd.json", @@ -99,7 +103,7 @@ public void verifyCorrectPageSizeDataReturn() throws IOException { // Given 6 records and we set page to 4, that means each query elastic return 4 record only // and the logic to load the reset can kick in. - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "5c418118-2581-4936-b6fd-d6bedfe74f62.json", "19da2ce7-138f-4427-89de-a50c724f5f54.json", "516811d7-cd1e-207a-e0440003ba8c79dd.json", @@ -187,7 +191,7 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { // Given 6 records and we set page to 4, that means each query elastic return 4 record only // and the logic to load the reset can kick in. - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "5c418118-2581-4936-b6fd-d6bedfe74f62.json", "19da2ce7-138f-4427-89de-a50c724f5f54.json", "516811d7-cd1e-207a-e0440003ba8c79dd.json", @@ -288,7 +292,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { // Given 6 records and we set page to 4, that means each query elastic return 4 record only // and the logic to load the reset can kick in. - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "5c418118-2581-4936-b6fd-d6bedfe74f62.json", "19da2ce7-138f-4427-89de-a50c724f5f54.json", "516811d7-cd1e-207a-e0440003ba8c79dd.json", @@ -358,7 +362,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { @Test public void verifyGetSingleCollection() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" ); @@ -377,7 +381,7 @@ public void verifyGetSingleCollection() throws IOException { @Test public void verifyBBoxCorrect() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "ae86e2f5-eaaf-459e-a405-e654d85adb9c.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" ); @@ -430,5 +434,129 @@ public void verifyBBoxCorrect() throws IOException { assertEquals(154.0, bbox.get(0).get(2).doubleValue(), "Overall bounding box coor 3"); assertEquals(-9.0, bbox.get(0).get(3).doubleValue(), "Overall bounding box coor 4"); } + /** + * Verify the function correctly sum up the values for feature id summary + * @throws IOException - Not expect to throw + */ + @Test + public void verifyAggregationFeatureSummaryCorrect() throws IOException { + super.insertJsonToElasticCODataIndex( + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.1.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json" + ); + + // Call rest api directly and get query result + ResponseEntity collection = testRestTemplate.getForEntity( + getBasePath() + "/collections/35234913-aa3c-48ec-b9a4-77f822f66ef8/items/summary", + FeatureCollectionGeoJSON.class); + + assertNotNull(collection.getBody(), "Body not null"); + + FeatureCollectionGeoJSON json = collection.getBody(); + assertEquals(3, json.getFeatures().size(), "Features correct"); + + // Sort make sure compare always same order + List sf = json.getFeatures().stream() + .sorted((a,b) -> b.getGeometry().hashCode() - a.getGeometry().hashCode()) + .toList(); + // Sample1 + FeatureGeoJSON featureGeoJSON1 = new FeatureGeoJSON(); + featureGeoJSON1.setType(FeatureGeoJSON.TypeEnum.FEATURE); + featureGeoJSON1.setGeometry(new PointGeoJSON().coordinates(List.of(BigDecimal.valueOf(159.26), BigDecimal.valueOf(-24.72)))); + featureGeoJSON1.setProperties(Map.of( + FeatureProperty.COUNT.getValue(), 42.0, + FeatureProperty.START_TIME.getValue(), "2023-02-01T00:00:00.000Z", + FeatureProperty.END_TIME.getValue(), "2023-02-01T00:00:00.000Z" + + )); + assertEquals(featureGeoJSON1, sf.get(0), "featureGeoJSON1"); + + // Sample3 + FeatureGeoJSON featureGeoJSON2 = new FeatureGeoJSON(); + featureGeoJSON2.setType(FeatureGeoJSON.TypeEnum.FEATURE); + featureGeoJSON2.setGeometry(new PointGeoJSON().coordinates(List.of(BigDecimal.valueOf(154.81), BigDecimal.valueOf(-26.2)))); + featureGeoJSON2.setProperties(Map.of( + FeatureProperty.COUNT.getValue(), 48.0, + FeatureProperty.START_TIME.getValue(), "2023-02-01T00:00:00.000Z", + FeatureProperty.END_TIME.getValue(), "2024-03-01T00:00:00.000Z" + + )); + assertEquals(featureGeoJSON2, sf.get(1), "featureGeoJSON2"); + + FeatureGeoJSON featureGeoJSON3 = new FeatureGeoJSON(); + featureGeoJSON3.setType(FeatureGeoJSON.TypeEnum.FEATURE); + featureGeoJSON3.setGeometry(new PointGeoJSON().coordinates(List.of(BigDecimal.valueOf(153.56), BigDecimal.valueOf(-26.59)))); + featureGeoJSON3.setProperties(Map.of( + FeatureProperty.COUNT.getValue(), 14.0, + FeatureProperty.START_TIME.getValue(), "2023-02-01T00:00:00.000Z", + FeatureProperty.END_TIME.getValue(), "2023-02-01T00:00:00.000Z" + + )); + assertEquals(featureGeoJSON3, sf.get(2), "featureGeoJSON3"); + } + /** + * We add more sample data and will trigger page load. + * @throws IOException - Not expect to throw + */ + @Test + public void verifyAggregationFeatureSummaryWithPageCorrect() throws IOException { + assertEquals(4, pageSize, "This test only works with small page"); + + super.insertJsonToElasticCODataIndex( + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.1.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample4.0.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.0.json", + "cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.1.json" + ); + + // Call rest api directly and get query result + ResponseEntity collection = testRestTemplate.getForEntity( + getBasePath() + "/collections/35234913-aa3c-48ec-b9a4-77f822f66ef8/items/summary", + FeatureCollectionGeoJSON.class); + + assertNotNull(collection.getBody(), "Body not null"); + + FeatureCollectionGeoJSON json = collection.getBody(); + assertEquals(5, json.getFeatures().size(), "Features correct"); + + // Sort make sure compare always same order + List sf = json.getFeatures().stream() + .sorted((a,b) -> b.getGeometry().hashCode() - a.getGeometry().hashCode()) + .toList(); + // Sample1 + FeatureGeoJSON featureGeoJSON1 = new FeatureGeoJSON(); + featureGeoJSON1.setType(FeatureGeoJSON.TypeEnum.FEATURE); + featureGeoJSON1.setGeometry(new PointGeoJSON().coordinates(List.of(BigDecimal.valueOf(163.56), BigDecimal.valueOf(-26.59)))); + featureGeoJSON1.setProperties(Map.of( + FeatureProperty.COUNT.getValue(), 14.0, + FeatureProperty.START_TIME.getValue(), "2023-02-01T00:00:00.000Z", + FeatureProperty.END_TIME.getValue(), "2023-02-01T00:00:00.000Z" + + )); + assertEquals(featureGeoJSON1, sf.get(0), "featureGeoJSON1"); + + // Sample5 + FeatureGeoJSON featureGeoJSON2 = new FeatureGeoJSON(); + featureGeoJSON2.setType(FeatureGeoJSON.TypeEnum.FEATURE); + featureGeoJSON2.setGeometry(new PointGeoJSON().coordinates(List.of(BigDecimal.valueOf(163.56), BigDecimal.valueOf(-126.59)))); + featureGeoJSON2.setProperties(Map.of( + FeatureProperty.COUNT.getValue(), 20.0, + FeatureProperty.START_TIME.getValue(), "2022-12-01T00:00:00.000Z", + FeatureProperty.END_TIME.getValue(), "2023-02-01T00:00:00.000Z" + + )); + assertEquals(featureGeoJSON2, sf.get(1), "featureGeoJSON2"); + } } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/tile/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/tile/RestApiTest.java index 4d7366e6..5efdb2ff 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/tile/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/tile/RestApiTest.java @@ -9,9 +9,6 @@ import java.io.IOException; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles("test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @@ -19,17 +16,17 @@ public class RestApiTest extends BaseTestClass { @BeforeAll - public void beforeClass() throws IOException { + public void beforeClass() { super.createElasticIndex(); } @AfterAll - public void clear() throws IOException { + public void clear() { super.clearElasticIndex(); } @BeforeEach - public void afterTest() throws IOException { + public void afterTest() { super.clearElasticIndex(); } @@ -43,7 +40,7 @@ public void verifyClusterIsHealthy() throws IOException { */ @Test public void verifyTiles() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" ); @@ -53,16 +50,18 @@ public void verifyTiles() throws IOException { InlineResponse2002.class ); - assertEquals(2, tiles.getBody().getTilesets().size(), "Count is correct"); - assertEquals("Impacts of stress on coral reproduction.", tiles.getBody().getTilesets().get(0).getTitle(), "Title matched 1"); - assertEquals("Ocean acidification historical reconstruction", tiles.getBody().getTilesets().get(1).getTitle(), "Title matched 2"); + Assertions.assertNotNull(tiles.getBody(), "Body not null"); + Assertions.assertNotNull(tiles.getBody().getTilesets(), "Tilesets not null"); + Assertions.assertEquals(2, tiles.getBody().getTilesets().size(), "Count is correct"); + Assertions.assertEquals("Impacts of stress on coral reproduction.", tiles.getBody().getTilesets().get(0).getTitle(), "Title matched 1"); + Assertions.assertEquals("Ocean acidification historical reconstruction", tiles.getBody().getTilesets().get(1).getTitle(), "Title matched 2"); } /** * Verify api call /collections/{collectionId}/tiles */ @Test public void verifyCollectionTiles() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "516811d7-cd1e-207a-e0440003ba8c79dd.json", "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" ); @@ -72,8 +71,10 @@ public void verifyCollectionTiles() throws IOException { InlineResponse2002.class ); - assertEquals(1, tiles.getBody().getTilesets().size(), "Count is correct"); - assertEquals("Impacts of stress on coral reproduction.", tiles.getBody().getTilesets().get(0).getTitle(), "Title matched 1"); + Assertions.assertNotNull(tiles.getBody(), "Body not null"); + Assertions.assertNotNull(tiles.getBody().getTilesets(), "Tilesets not null"); + Assertions.assertEquals(1, tiles.getBody().getTilesets().size(), "Count is correct"); + Assertions.assertEquals("Impacts of stress on coral reproduction.", tiles.getBody().getTilesets().get(0).getTitle(), "Title matched 1"); } /** * Verify api call /tiles/{tileMatrixSetId}/{tileMatrix}/{tileRow}/{tileCol}, this call will return the bounding @@ -82,7 +83,7 @@ public void verifyCollectionTiles() throws IOException { */ @Test public void verifyTilesMatrixSetXYZ() throws IOException { - super.insertJsonToElasticIndex( + super.insertJsonToElasticRecordIndex( "b299cdcd-3dee-48aa-abdd-e0fcdbb9cadc.json" ); diff --git a/server/src/test/resources/data_index_schema.json b/server/src/test/resources/data_index_schema.json new file mode 100644 index 00000000..28ad4c00 --- /dev/null +++ b/server/src/test/resources/data_index_schema.json @@ -0,0 +1,46 @@ +{ + "mappings": { + "properties": { + "id": { + "type": "keyword" + }, + "stac_version": { "type": "text" }, + "stac_extensions": { "type": "text" }, + "type": { "type": "text" }, + "collection": { + "type": "keyword" + }, + "geometry": { + "type": "geo_shape" + }, + "bbox" : { + "type": "double" + }, + "properties" : { + "type": "nested", + "properties" : { + "lng": { "type": "double" }, + "lat": { "type": "double" }, + "depth": { "type": "double" }, + "count": { "type": "double" }, + "time": { "type": "date" } + } + }, + "links": { + "type": "nested", + "properties": { + "link" : { + "type": "nested", + "properties": { + "href": { "type": "text" }, + "rel": { "type": "text" }, + "type": { "type": "text" }, + "title": { "type": "text" }, + "description": { "type": "text" } + } + } + } + } + } + } +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json new file mode 100644 index 00000000..f4e3a328 --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.0.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|159.26|-24.72|150.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 159.26, + -24.72 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 150, + "lng": 159.26, + "lat": -24.72, + "time": "2023-02-01T00:00:00Z", + "count": 14 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.1.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.1.json new file mode 100644 index 00000000..985967bf --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.1.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|159.26|-24.72|940.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 159.26, + -24.72 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 940, + "lng": 159.26, + "lat": -24.72, + "time": "2023-02-01T00:00:00Z", + "count": 13 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json new file mode 100644 index 00000000..1c004c5b --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample1.2.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|159.26|-24.72|570.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 159.26, + -24.72 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 570, + "lng": 159.26, + "lat": -24.72, + "time": "2023-02-01T00:00:00Z", + "count": 15 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json new file mode 100644 index 00000000..e9bf727e --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample2.0.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|153.56|-26.59|760.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 153.56, + -26.59 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 760, + "lng": 153.56, + "lat": -26.59, + "time": "2023-02-01T00:00:00Z", + "count": 14 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json new file mode 100644 index 00000000..1653dbf4 --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.0.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|154.81|-26.20|260.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 154.81, + -26.2 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 260, + "lng": 154.81, + "lat": -26.2, + "time": "2023-02-01T00:00:00Z", + "count": 14 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json new file mode 100644 index 00000000..b32c7662 --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.1.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2024-03|154.81|-26.20|260.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 154.81, + -26.2 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 260, + "lng": 154.81, + "lat": -26.2, + "time": "2024-03-01T00:00:00Z", + "count": 20 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json new file mode 100644 index 00000000..071b94d8 --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample3.2.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|154.81|-26.20|260.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 154.81, + -26.2 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 260, + "lng": 154.81, + "lat": -26.2, + "time": "2023-02-01T00:00:00Z", + "count": 14 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample4.0.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample4.0.json new file mode 100644 index 00000000..cc8e5756 --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample4.0.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|163.56|-26.59|760.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 163.56, + -26.59 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 760, + "lng": 163.56, + "lat": -26.59, + "time": "2023-02-01T00:00:00Z", + "count": 14 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.0.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.0.json new file mode 100644 index 00000000..1b05f1e4 --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.0.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|163.56|-126.59|760.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 163.56, + -126.59 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 760, + "lng": 163.56, + "lat": -126.59, + "time": "2023-02-01T00:00:00Z", + "count": 10 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} diff --git a/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.1.json b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.1.json new file mode 100644 index 00000000..f2e946f5 --- /dev/null +++ b/server/src/test/resources/databag/cloudoptimized/35234913-aa3c-48ec-b9a4-77f822f66ef8/sample5.1.json @@ -0,0 +1,31 @@ +{ + "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2023-02|163.56|-126.59|760.00", + "geometry": { + "geometries": [{ + "coordinates": [ + 163.56, + -126.59 + ], + "type": "Point" + }], + "type": "GeometryCollection" + }, + "properties": { + "depth": 760, + "lng": 163.56, + "lat": -126.59, + "time": "2022-12-01T00:00:00Z", + "count": 10 + }, + "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/language/v1.0.0/schema.json", + "https://stac-extensions.github.io/themes/v1.0.0/schema.json", + "https://stac-extensions.github.io/web-map-links/v1.2.0/schema.json" + ], + "type": "Feature" +} diff --git a/server/src/test/resources/portal_records_index_schema.json b/server/src/test/resources/portal_records_index_schema.json index e85f05d4..1600c8c3 100644 --- a/server/src/test/resources/portal_records_index_schema.json +++ b/server/src/test/resources/portal_records_index_schema.json @@ -88,6 +88,9 @@ "organisation_vocabs_sayt": { "type": "search_as_you_type", "analyzer": "custom_analyser" } } }, + "parameter_vocabs": { "type": "keyword" }, + "platform_vocabs": { "type": "keyword" }, + "organisation_vocabs": { "type": "keyword" }, "keywords": { "type": "nested", "properties": { @@ -162,9 +165,6 @@ } } }, - "parameter_vocabs": { "type": "keyword" }, - "platform_vocabs": { "type": "keyword" }, - "organisation_vocabs": { "type": "keyword" }, "statement": { "type": "text" } } },