Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
cache: 'maven'

- name: Build with Maven
run: mvn -B verify --file pom.xml
run: mvn clean -B verify --file pom.xml

- name: Build Docker Image
uses: docker/build-push-action@v5
Expand Down
29 changes: 14 additions & 15 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,17 @@ VOLUME /tmp
ENV MAX_HEAP_PERCENTAGE=70

COPY ./server/target/ogcapi-java-server-*-exec.jar app.jar
ENTRYPOINT [\
"java",\
"-XX:MaxRAMPercentage=${MAX_HEAP_PERCENTAGE}",\
"-Delasticsearch.index.name=${INDEX_NAME}",\
"-Delasticsearch.cloud_optimized_index.name=${CO_INDEX_NAME}",\
"-Delasticsearch.vocabs_index.name=${VOCABS_INDEX_NAME}",\
"-Dapi.host=${HOST}:${PORT}",\
"-Dserver.port=${PORT}",\
"-Delasticsearch.serverUrl=${ELASTIC_URL}",\
"-Delasticsearch.apiKey=${ELASTIC_KEY}",\
"-Ddata-access-service.host=${DAS_HOST}",\
"-Ddata-access-service.secret=${DAS_SECRET}",\
"--enable-preview",\
"-jar",\
"/app.jar"]
ENTRYPOINT ["/bin/sh", "-c", "java \
-XX:MaxRAMPercentage=${MAX_HEAP_PERCENTAGE} \
-Delasticsearch.index.name=${INDEX_NAME} \
-Delasticsearch.cloud_optimized_index.name=${CO_INDEX_NAME} \
-Delasticsearch.vocabs_index.name=${VOCABS_INDEX_NAME} \
-Dapi.host=${HOST}:${PORT} \
-Dserver.port=${PORT} \
-Delasticsearch.serverUrl=${ELASTIC_URL} \
-Delasticsearch.apiKey=${ELASTIC_KEY} \
-Ddata-access-service.host=${DAS_HOST} \
-Ddata-access-service.secret=${DAS_SECRET} \
--enable-preview \
-jar \
/app.jar"]
3 changes: 3 additions & 0 deletions server/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,9 @@
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<argLine>--enable-preview</argLine>
<systemPropertyVariables>
<spring.profiles.active>test</spring.profiles.active> <!-- e.g., for Spring Boot -->
</systemPropertyVariables>
</configuration>
</plugin>
<plugin>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,8 @@ public enum CQLFields implements CQLFieldsInterface {
CQLElasticSetting.score.getSetting(),
CQLElasticSetting.score.getSetting(),
null,
(order) -> new SortOptions.Builder().field(f -> f.field(CQLElasticSetting.score.getSetting()).order(order))
(order) -> new SortOptions.Builder()
.field(f -> f.field(CQLElasticSetting.score.getSetting()).order(order))
),
// Rank score is an internal calculated score, it is different from the one use by ElasticSearch,
// @see es-indexer RankingService
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,7 @@ public ElasticSearchBase.SearchResult<StacCollectionModel> searchByParameters(Li
}
}
catch(Exception e) {
log.warn("Error parsing score assume null", e);
// OK to ignore as accept null as the value
}
// Get the search after
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/resources/log4j2-spring.xml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
</Console>
</Appenders>
<Loggers>
<Logger name="au.org.aodn.esindexer" level="debug" additivity="false">
<Logger name="au.org.aodn.ogcapi" level="debug" additivity="false">
<AppenderRef ref="DevConsole"/>
</Logger>
<Root level="info">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders;
import co.elastic.clients.elasticsearch.core.*;
import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem;
import co.elastic.clients.elasticsearch.core.search.Hit;
import co.elastic.clients.elasticsearch.indices.CreateIndexRequest;
import co.elastic.clients.transport.rest_client.RestClientTransport;
import com.fasterxml.jackson.databind.JsonNode;
Expand Down Expand Up @@ -241,6 +242,11 @@ protected void insertJsonToElasticIndex(String index, String[] filenames) throws
logger.debug(response.toString());

assertEquals(filenames.length, response.hits().hits().size(), "Number of docs stored is correct");
for (Hit<ObjectNode> hit : response.hits().hits()) {
if(hit.source() != null) {
logger.debug("Stored the following id {}", hit.source().get("id"));
}
}
}

protected void insertJsonToElasticRecordIndex(String... filenames) throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ public void clear() {
@BeforeEach
public void afterTest() {
super.clearElasticIndex();
super.createElasticIndex();
}

@Test
Expand Down Expand Up @@ -290,6 +291,8 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException {
public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException {
assertEquals(4, pageSize, "This test only works with small page");

logger.debug("Start verifyCorrectPageSizeAndScoreWithQuery");

// Given 6 records and we set page to 4, that means each query elastic return 4 record only
// and the logic to load the reset can kick in.
super.insertJsonToElasticRecordIndex(
Expand All @@ -308,6 +311,8 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException {
new ParameterizedTypeReference<>() {
});

logger.debug("verifyCorrectPageSizeAndScoreWithQuery - Done query 1");

assertEquals(HttpStatus.OK, collections.getStatusCode(), "Get status OK");
// Given request page size is 1
assertEquals(1,
Expand All @@ -319,6 +324,10 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException {

// The search after give you the value to go to next batch
assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields");

logger.debug("verifyCorrectPageSizeAndScoreWithQuery - uuid return {}", collections.getBody().getCollections().get(0).getId());
logger.debug("verifyCorrectPageSizeAndScoreWithQuery - search after {}", collections.getBody().getSearchAfter());

assertEquals(
"80",
collections.getBody().getSearchAfter().get(1),
Expand All @@ -343,6 +352,8 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException {
new ParameterizedTypeReference<>() {
});

logger.debug("Start verifyCorrectPageSizeAndScoreWithQuery - Done query 2");

assertEquals(HttpStatus.OK, collections.getStatusCode(), "Get status OK");
assertEquals(4,
Objects.requireNonNull(collections.getBody()).getCollections().size(),
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"title": "IMOS - National Reef Monitoring Network Sub-Facility",
"description": "The National Reef Monitoring Network brings together shallow reef surveys conducted around Australia into a centralised database. The IMOS National Reef Monitoring Network sub-Facility collates, cleans, stores and makes this data rapidly available from contributors including: Reef Life Survey, Parks Australia, Department of Biodiversity, Conservation and Attractions (Western Australia), Department of Environment, Water and Natural Resources (South Australia), Department of Primary Industries (New South Wales), Tasmanian Parks and Wildlife Service and Parks Victoria. The data provided by the National Reef Monitoring Network contributes to establishing and supporting national marine baselines, and assisting with the management of Commonwealth and State marine reserves. Reef Life Survey (RLS) and the Australian Temperate Reef Network (ATRC) aims to improve biodiversity conservation and the sustainable management of marine resources by coordinating surveys of rocky and coral reefs using scientific methods, with the ultimate goal to improve coastal stewardship. Our activities depend on the skills of marine scientists, experienced and motivated recreational SCUBA divers, partnerships with management agencies and university researchers, and active input from the ATRC partners and RLS Advisory Committee RLS and ATRC data are freely available to the public for non-profit purposes, so not only managers, but also groups such as local dive clubs or schools may use these data to look at changes over time in their own local reefs. By making data freely available and through public outputs, RLS and ATRC aims to raise broader community awareness of the status of Australia’s marine biodiversity and associated conservation issues.",
"description": "No value is fine, this case is make sure short form of National Reef Monitoring Network not appear here",
"extent": {
"bbox": [
[
Expand Down Expand Up @@ -40,7 +40,7 @@
},
"creation": "2021-05-21T12:00:00",
"revision": "2024-07-04T03:50:26",
"ai:description": "The National Reef Monitoring Network brings together shallow reef surveys conducted around Australia into a centralised database. The IMOS National Reef Monitoring Network sub-Facility collates, cleans, stores and makes this data rapidly available from contributors including: - Reef Life Surve - Parks Australi - Department of Biodiversity, Conservation and Attractions (Western Australia) - Department of Environment, Water and Natural Resources (South Australia) - Department of Primary Industries (New South Wales) - Tasmanian Parks and Wildlife Service - Parks Victoria. The data provided by the National Reef Monitoring Network contributes to establishing and supporting national marine baselines, and assisting with the management of Commonwealth and State marine reserves. Reef Life Survey (RLS) and the Australian Temperate Reef Network (ATRC) aims to improve biodiversity conservation and the sustainable management of marine resources by coordinating surveys of rocky and coral reefs using scientific methods, with the ultimate goal to improve coastal stewardship. Our activities depend on the skills of marine scientists, experienced and motivated recreational SCUBA divers, partnerships with management agencies and university researchers, and active input from the ATRC partners and RLS Advisory Committee. RLS and ATRC data are freely available to the public for non-profit purposes, so not only managers, but also groups such as local dive clubs or schools may use these data to look at changes over time in their own local reefs. By making data freely available and through public outputs, RLS and ATRC aims to raise broader community awareness of the status of Australia’s marine biodiversity and associated conservation issues.",
"ai:description": "No value is fine",
"dataset_provider": "IMOS",
"update_frequency": "other",
"proj:geometry": {
Expand Down Expand Up @@ -11300,8 +11300,7 @@
"href": "uuid:0f65b7ae-1f6f-4a55-b804-1c991f791e1a",
"rel": "sibling",
"type": "application/json",
"title": "{\"title\":\"IMOS - Autonomous Underwater Vehicles - AUV Iver",
"recordAbstract": "The IMOS Autonomous Underwater Vehicles Facility has an Autonomous Underwater Vehicle (AUV) called Iver (IMOS platform code:IVER), capable of undertaking high resolution geo-referenced survey work. This platform is a modified Ocean Server Iver2 AUV that is hand deployable off RHIBs and other small vessels in addition to being deployable off larger vessels.\n\nThis AUV has been modified for benthic imaging, including the addition of USBL and DVL for more accurate navigation and the addition of high resolution stereo cameras and strobes. The submersible is equipped with a suite of oceanographic sensors including high resolution stereo cameras (6MP each), depth sensor, Doppler Velocity Log (DVL), Compass and Ultra Short Baseline (USBL). The vehicle is controlled by an on-board PC stack which is used to log sensor information and run the vehicle's control algorithms.\n\nThe vehicle has demonstrated its capacity to collect high resolution, near bottom imagery on trajectories over smooth terrain that has been used to generate 3D meshes and ortho-mosaics.\"}"
"title": "Any works"
}
],
"license": "Creative Commons Attribution 4.0 International License",
Expand Down Expand Up @@ -11793,7 +11792,7 @@
"National Reef Monitoring Network Sub-Facility, Integrated Marine Observing System (IMOS)"
]
},
"sci:citation": "{\"suggestedCitation\":\"The citation in a list of references is: \\\"Reef Life Survey (RLS); Institute for Marine and Antarctic Studies (IMAS); Parks Victoria; Department of Primary Industries (DPI), New South Wales Government; Parks and Wildlife Tasmania; Department for Environment and Water (DEWNR), South Australia, Integrated Marine Observing System (IMOS) [year-of-data-download], National Reef Monitoring Network Sub-Facility, [data-access-URL], accessed [date-of-access].\\\"\",\"useLimitations\":[\"Data, products and services from IMOS are provided \\\"as is\\\" without any warranty as to fitness for a particular purpose.\"],\"otherConstraints\":null}",
"sci:citation": "Not importand for testing",
"type": "Collection",
"stac_version": "1.0.0",
"stac_extensions": [
Expand Down
6 changes: 3 additions & 3 deletions server/src/test/resources/portal_records_index_schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,9 @@
},
"summaries": {
"properties": {
"ai:description": {
"type": "text"
},
"score": {
"type": "long"
},
Expand Down Expand Up @@ -253,9 +256,6 @@
},
"statement": {
"type": "text"
},
"ai:description": {
"type": "text"
}
}
},
Expand Down