From 502ff1381efdbebaf797dfca4a1f2f0aa62c0f81 Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Thu, 14 Nov 2024 11:26:33 +1100 Subject: [PATCH 1/3] summarizer to optimize the data size --- .../server/core/util/DatasetSummarizer.java | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/util/DatasetSummarizer.java diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/DatasetSummarizer.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/DatasetSummarizer.java new file mode 100644 index 00000000..0398febd --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/DatasetSummarizer.java @@ -0,0 +1,164 @@ +package au.org.aodn.ogcapi.server.core.util; + +import au.org.aodn.ogcapi.features.model.FeatureCollectionGeoJSON; +import au.org.aodn.ogcapi.features.model.FeatureGeoJSON; +import au.org.aodn.ogcapi.features.model.PointGeoJSON; +import au.org.aodn.ogcapi.server.core.model.enumeration.FeatureProperty; +import lombok.Getter; + +import java.time.LocalDate; +import java.util.HashMap; +import java.util.Map; + +public class DatasetSummarizer { + + private final FeatureCollectionGeoJSON dataset; + + @Getter + private final FeatureCollectionGeoJSON summarizedDataset; + + public DatasetSummarizer(FeatureCollectionGeoJSON dataset) { + this.dataset = dataset; + this.summarizedDataset = new FeatureCollectionGeoJSON(); + this.summarizedDataset.setType(FeatureCollectionGeoJSON.TypeEnum.FEATURECOLLECTION); + summarizeDataset(); + } + + + public void summarizeDataset() { + for (var feature : dataset.getFeatures()) { + var sameLocationFeature = getSameLocationFeature(feature); + if (sameLocationFeature == null) { + summarizedDataset + .getFeatures() + .add(generateNewSummarizedFeature(feature)); + continue; + } + var newFeature = aggregateFeature(sameLocationFeature, feature); + summarizedDataset.getFeatures().remove(sameLocationFeature); + summarizedDataset.getFeatures().add(newFeature); + } + } + + @SuppressWarnings("unchecked") + private FeatureGeoJSON aggregateFeature(FeatureGeoJSON existingFeature, FeatureGeoJSON featureToAdd) { + var aggregatedFeature = new FeatureGeoJSON(); + aggregatedFeature.setType(FeatureGeoJSON.TypeEnum.FEATURE); + aggregatedFeature.setGeometry(existingFeature.getGeometry()); + + try{ + var existingProperties = (Map) existingFeature.getProperties(); + + var newTimeStr = getTime(featureToAdd); + existingProperties = updateTimeRange(existingProperties, newTimeStr); + + var newCount = (Long) getPropertyFromFeature(featureToAdd, FeatureProperty.COUNT); + existingProperties = updateCount(existingProperties, newCount); + + aggregatedFeature.setProperties(existingProperties); + } catch (ClassCastException e) { + throw new RuntimeException("Feature properties is not a map", e); + } + return aggregatedFeature; + } + + private Map updateCount(Map existingProperties, Long newCount) { + var existingCount = (Long) existingProperties.get(FeatureProperty.COUNT.getValue()); + var updatedProperties = new HashMap<>(existingProperties); + updatedProperties.put(FeatureProperty.COUNT.getValue(), existingCount + newCount); + return updatedProperties; + } + + private Map updateTimeRange(Map existingProperties, String newTimeStr) { + + var startTimeStr = (String) existingProperties.get(FeatureProperty.START_TIME.getValue()); + var endTimeStr = (String) existingProperties.get(FeatureProperty.END_TIME.getValue()); + + var updatedProperties = new HashMap<>(existingProperties); + updatedProperties.remove(FeatureProperty.TIME.getValue()); + if (newTimeStr != null) { + var newTime = LocalDate.parse(newTimeStr); + var startTime = LocalDate.parse(startTimeStr); + var endTime = LocalDate.parse(endTimeStr); + if (newTime.isBefore(startTime)) { + startTimeStr = newTimeStr; + } else if (newTime.isAfter(endTime)) { + endTimeStr = newTimeStr; + } + updatedProperties.put(FeatureProperty.START_TIME.getValue(), startTimeStr); + updatedProperties.put(FeatureProperty.END_TIME.getValue(), endTimeStr); + } + return updatedProperties; + } + + + private FeatureGeoJSON generateNewSummarizedFeature(FeatureGeoJSON feature) { + var summarizedFeature = new FeatureGeoJSON(); + summarizedFeature.setType(FeatureGeoJSON.TypeEnum.FEATURE); + summarizedFeature.setGeometry(feature.getGeometry()); + summarizedFeature.setProperties(feature.getProperties()); + var time = getTime(feature); + setPropertyToFeature(feature, FeatureProperty.START_TIME , time); + setPropertyToFeature(feature, FeatureProperty.END_TIME , time); + + return summarizedFeature; + } + + @SuppressWarnings("unchecked") + private void setPropertyToFeature( + FeatureGeoJSON feature, FeatureProperty propertyType, String value + ) { + try{ + var properties = (Map) feature.getProperties(); + properties.put(propertyType.getValue(), value); + } catch (ClassCastException e) { + throw new RuntimeException("Feature properties is not a map", e); + } + } + + @SuppressWarnings("unchecked") + private Object getPropertyFromFeature(FeatureGeoJSON feature, FeatureProperty propertyType) { + try{ + var properties = (Map) feature.getProperties(); + return properties.get(propertyType.getValue()); + } catch (ClassCastException e) { + throw new RuntimeException("Feature properties is not a map", e); + } + } + + + private FeatureGeoJSON getSameLocationFeature(FeatureGeoJSON feature) { + for (var f : summarizedDataset.getFeatures()) { + try { + var existingCoordinates = ((PointGeoJSON) f.getGeometry()).getCoordinates(); + var newCoordinates = ((PointGeoJSON) feature.getGeometry()).getCoordinates(); + if (existingCoordinates.get(0).equals(newCoordinates.get(0)) && existingCoordinates.get(1).equals(newCoordinates.get(1))) { + return f; + } + } catch (ClassCastException e) { + throw new RuntimeException("Feature geometry is not a point", e); + } + } + return null; + } + + @SuppressWarnings("unchecked") + private String getTime(FeatureGeoJSON feature) { + if (feature.getProperties() instanceof Map) { + var properties = (Map) feature.getProperties(); + return (String) properties.get(FeatureProperty.TIME.getValue()); + } + return null; + } +} + +// following are values from experience. can change +//1 decimal place: ~11.1 kilometers +//2 decimal places: ~1.1 kilometers +//3 decimal places: ~110 meters +//4 decimal places: ~11 meters +//5 decimal places: ~1.1 meters + +// zoom level less than 7.3, 1 decimal place +// zoom level less than 4, integer +// zoom level less than 1.2, 10 \ No newline at end of file From f97ad618f8afe48b1e220e72c5f04a2f3ebfbb3d Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Wed, 20 Nov 2024 11:43:01 +1100 Subject: [PATCH 2/3] temp commit --- .../core/model/DatasetSearchResult.java | 13 ++++-- .../model/enumeration/FeatureProperty.java | 5 ++- .../server/core/util/DatasetSummarizer.java | 41 +++++++++++-------- .../aodn/ogcapi/server/features/RestApi.java | 7 +++- .../ogcapi/server/features/RestServices.java | 4 +- 5 files changed, 46 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetSearchResult.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetSearchResult.java index 3da6ffe9..37e0a1c0 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetSearchResult.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetSearchResult.java @@ -1,15 +1,16 @@ package au.org.aodn.ogcapi.server.core.model; -import au.org.aodn.ogcapi.features.model.*; +import au.org.aodn.ogcapi.features.model.FeatureCollectionGeoJSON; +import au.org.aodn.ogcapi.features.model.FeatureGeoJSON; +import au.org.aodn.ogcapi.features.model.PointGeoJSON; import au.org.aodn.ogcapi.server.core.model.enumeration.FeatureProperty; -import lombok.Getter; +import au.org.aodn.ogcapi.server.core.util.DatasetSummarizer; import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; -@Getter public class DatasetSearchResult { private final FeatureCollectionGeoJSON dataset; @@ -24,6 +25,12 @@ private void initDataset() { dataset.setFeatures(new ArrayList<>()); } + public FeatureCollectionGeoJSON getSummarizedDataset() { + var summarizer = new DatasetSummarizer(dataset); + summarizer.summarizeDataset(); + return summarizer.getSummarizedDataset(); + } + public void addDatum(DatumModel datum) { if (datum == null) { diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureProperty.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureProperty.java index e9a00b87..441cf56e 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureProperty.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureProperty.java @@ -7,7 +7,10 @@ public enum FeatureProperty { TIME("time"), DEPTH("depth"), COUNT("count"), - UUID("uuid") + UUID("uuid"), + START_TIME("startTime"), + END_TIME("endTime"), + COORDINATE_ACCURACY("coordinateAccuracy"), ; private final String value; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/DatasetSummarizer.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/DatasetSummarizer.java index 0398febd..ba42ecea 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/DatasetSummarizer.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/DatasetSummarizer.java @@ -6,7 +6,7 @@ import au.org.aodn.ogcapi.server.core.model.enumeration.FeatureProperty; import lombok.Getter; -import java.time.LocalDate; +import java.time.YearMonth; import java.util.HashMap; import java.util.Map; @@ -50,12 +50,12 @@ private FeatureGeoJSON aggregateFeature(FeatureGeoJSON existingFeature, FeatureG var existingProperties = (Map) existingFeature.getProperties(); var newTimeStr = getTime(featureToAdd); - existingProperties = updateTimeRange(existingProperties, newTimeStr); + var timeUpdatedProperties = updateTimeRange(existingProperties, newTimeStr); - var newCount = (Long) getPropertyFromFeature(featureToAdd, FeatureProperty.COUNT); - existingProperties = updateCount(existingProperties, newCount); + var newCount = getCount(featureToAdd); + var timeAndCountUpdatedProperties = updateCount(timeUpdatedProperties, newCount); - aggregatedFeature.setProperties(existingProperties); + aggregatedFeature.setProperties(timeAndCountUpdatedProperties); } catch (ClassCastException e) { throw new RuntimeException("Feature properties is not a map", e); } @@ -77,9 +77,9 @@ private Map updateTimeRange(Map existingProperti var updatedProperties = new HashMap<>(existingProperties); updatedProperties.remove(FeatureProperty.TIME.getValue()); if (newTimeStr != null) { - var newTime = LocalDate.parse(newTimeStr); - var startTime = LocalDate.parse(startTimeStr); - var endTime = LocalDate.parse(endTimeStr); + var newTime = YearMonth.parse(newTimeStr); + var startTime = YearMonth.parse(startTimeStr); + var endTime = YearMonth.parse(endTimeStr); if (newTime.isBefore(startTime)) { startTimeStr = newTimeStr; } else if (newTime.isAfter(endTime)) { @@ -126,6 +126,22 @@ private Object getPropertyFromFeature(FeatureGeoJSON feature, FeatureProperty pr } } + private Long getCount(FeatureGeoJSON feature) { + try { + return (Long) getPropertyFromFeature(feature, FeatureProperty.COUNT); + } catch (Exception e) { + throw new RuntimeException("failed to get count from feature", e); + } + } + + private String getTime(FeatureGeoJSON feature) { + try { + return (String) getPropertyFromFeature(feature, FeatureProperty.TIME); + } catch (Exception e) { + throw new RuntimeException("failed to get time from feature", e); + } + } + private FeatureGeoJSON getSameLocationFeature(FeatureGeoJSON feature) { for (var f : summarizedDataset.getFeatures()) { @@ -142,14 +158,7 @@ private FeatureGeoJSON getSameLocationFeature(FeatureGeoJSON feature) { return null; } - @SuppressWarnings("unchecked") - private String getTime(FeatureGeoJSON feature) { - if (feature.getProperties() instanceof Map) { - var properties = (Map) feature.getProperties(); - return (String) properties.get(FeatureProperty.TIME.getValue()); - } - return null; - } + } // following are values from experience. can change diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java index 92b93c3f..64d01096 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java @@ -40,9 +40,12 @@ public ResponseEntity getFeatures( @PathVariable("collectionId") String collectionId, @RequestParam(value = "start_datetime", required = false) String startDate, - @RequestParam(value = "end_datetime", required = false) String endDate + @RequestParam(value = "end_datetime", required = false) String endDate, + // keep these two parameters for future usage + @RequestParam(value= "zoom", required = false) Double zoomLevel, + @RequestParam(value="bbox", required = false) List bbox ) { - return featuresService.getDataset(collectionId, startDate, endDate); + return featuresService.getSummarizedDataset(collectionId, startDate, endDate); } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java index e35ead31..41c81fc6 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java @@ -41,7 +41,7 @@ public ResponseEntity getCollection(String id, String sortBy) throws } } - public ResponseEntity getDataset( + public ResponseEntity getSummarizedDataset( String collectionId, String startDate, String endDate @@ -49,7 +49,7 @@ public ResponseEntity getDataset( try { var result = search.searchDataset(collectionId, startDate, endDate); return ResponseEntity.ok() - .body(result.getDataset()); + .body(result.getSummarizedDataset()); } catch (Exception e) { log.error("Error while getting dataset", e); return ResponseEntity.internalServerError().build(); From 46c47461aff0ae3d3417f5c32a5b124ac256785d Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Wed, 20 Nov 2024 13:21:13 +1100 Subject: [PATCH 3/3] pre-commit test fix --- .../au/org/aodn/ogcapi/server/core/util/DatasetSummarizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/DatasetSummarizer.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/DatasetSummarizer.java index ba42ecea..207154d5 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/DatasetSummarizer.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/DatasetSummarizer.java @@ -170,4 +170,4 @@ private FeatureGeoJSON getSameLocationFeature(FeatureGeoJSON feature) { // zoom level less than 7.3, 1 decimal place // zoom level less than 4, integer -// zoom level less than 1.2, 10 \ No newline at end of file +// zoom level less than 1.2, 10