From 30207c5d68d001839e576f979680c282cfbaa51b Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Tue, 14 Jan 2025 15:31:55 +1100 Subject: [PATCH 01/23] new customized process files --- .../aodn/ogcapi/processes/common/RestApi.java | 48 +++++++++++++++++++ .../processes/processes/ProcessIdEnum.java | 17 +++++++ 2 files changed, 65 insertions(+) create mode 100644 processes/src/main/java/au/org/aodn/ogcapi/processes/common/RestApi.java create mode 100644 processes/src/main/java/au/org/aodn/ogcapi/processes/processes/ProcessIdEnum.java diff --git a/processes/src/main/java/au/org/aodn/ogcapi/processes/common/RestApi.java b/processes/src/main/java/au/org/aodn/ogcapi/processes/common/RestApi.java new file mode 100644 index 00000000..79fc8473 --- /dev/null +++ b/processes/src/main/java/au/org/aodn/ogcapi/processes/common/RestApi.java @@ -0,0 +1,48 @@ +package au.org.aodn.ogcapi.processes.common; + +import au.org.aodn.ogcapi.processes.api.JobsApi; +import au.org.aodn.ogcapi.processes.api.ProcessesApi; +import au.org.aodn.ogcapi.processes.model.Process; +import au.org.aodn.ogcapi.processes.model.*; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController("CommonRestApi") +@RequestMapping(value = "/api/v1/ogc") +public class RestApi implements ProcessesApi, JobsApi { + @Override + public ResponseEntity execute(String processID, Execute body) { + return null; + } + + @Override + public ResponseEntity getProcessDescription(String processID) { + return null; + } + + @Override + public ResponseEntity getProcesses() { + return null; + } + + @Override + public ResponseEntity dismiss(String jobId) { + return null; + } + + @Override + public ResponseEntity getJobs() { + return null; + } + + @Override + public ResponseEntity getResult(String jobId) { + return null; + } + + @Override + public ResponseEntity getStatus(String jobId) { + return null; + } +} diff --git a/processes/src/main/java/au/org/aodn/ogcapi/processes/processes/ProcessIdEnum.java b/processes/src/main/java/au/org/aodn/ogcapi/processes/processes/ProcessIdEnum.java new file mode 100644 index 00000000..8f2fce37 --- /dev/null +++ b/processes/src/main/java/au/org/aodn/ogcapi/processes/processes/ProcessIdEnum.java @@ -0,0 +1,17 @@ +package au.org.aodn.ogcapi.processes.processes; + +import lombok.Getter; + +@Getter +public enum ProcessIdEnum { + GENERATE_CDF("generate-cdf"), + + ; + + private final String value; + + ProcessIdEnum(String value) { + this.value = value; + } + +} From d4f9095050485ad1574d365f695b29be8a81f4da Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Thu, 16 Jan 2025 11:01:45 +1100 Subject: [PATCH 02/23] change the location of the api implementation --- .../aodn/ogcapi/processes/common/RestApi.java | 48 ------------------- .../processes/processes/ProcessIdEnum.java | 17 ------- .../aodn/ogcapi/server/processes/RestApi.java | 28 +++++++++++ 3 files changed, 28 insertions(+), 65 deletions(-) delete mode 100644 processes/src/main/java/au/org/aodn/ogcapi/processes/common/RestApi.java delete mode 100644 processes/src/main/java/au/org/aodn/ogcapi/processes/processes/ProcessIdEnum.java create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java diff --git a/processes/src/main/java/au/org/aodn/ogcapi/processes/common/RestApi.java b/processes/src/main/java/au/org/aodn/ogcapi/processes/common/RestApi.java deleted file mode 100644 index 79fc8473..00000000 --- a/processes/src/main/java/au/org/aodn/ogcapi/processes/common/RestApi.java +++ /dev/null @@ -1,48 +0,0 @@ -package au.org.aodn.ogcapi.processes.common; - -import au.org.aodn.ogcapi.processes.api.JobsApi; -import au.org.aodn.ogcapi.processes.api.ProcessesApi; -import au.org.aodn.ogcapi.processes.model.Process; -import au.org.aodn.ogcapi.processes.model.*; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -@RestController("CommonRestApi") -@RequestMapping(value = "/api/v1/ogc") -public class RestApi implements ProcessesApi, JobsApi { - @Override - public ResponseEntity execute(String processID, Execute body) { - return null; - } - - @Override - public ResponseEntity getProcessDescription(String processID) { - return null; - } - - @Override - public ResponseEntity getProcesses() { - return null; - } - - @Override - public ResponseEntity dismiss(String jobId) { - return null; - } - - @Override - public ResponseEntity getJobs() { - return null; - } - - @Override - public ResponseEntity getResult(String jobId) { - return null; - } - - @Override - public ResponseEntity getStatus(String jobId) { - return null; - } -} diff --git a/processes/src/main/java/au/org/aodn/ogcapi/processes/processes/ProcessIdEnum.java b/processes/src/main/java/au/org/aodn/ogcapi/processes/processes/ProcessIdEnum.java deleted file mode 100644 index 8f2fce37..00000000 --- a/processes/src/main/java/au/org/aodn/ogcapi/processes/processes/ProcessIdEnum.java +++ /dev/null @@ -1,17 +0,0 @@ -package au.org.aodn.ogcapi.processes.processes; - -import lombok.Getter; - -@Getter -public enum ProcessIdEnum { - GENERATE_CDF("generate-cdf"), - - ; - - private final String value; - - ProcessIdEnum(String value) { - this.value = value; - } - -} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java new file mode 100644 index 00000000..6d5c3d25 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java @@ -0,0 +1,28 @@ +package au.org.aodn.ogcapi.server.processes; + + +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import jakarta.validation.Valid; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController("ProcessesRestApi") +@RequestMapping(value = "/api/v1/ogc") +public class RestApi { + + @PostMapping(path="/processes/download/{collectionId}") + public ResponseEntity downloadData( + @PathVariable("collectionId") String collectionId, + @Parameter(in = ParameterIn.QUERY, description = "start date") @Valid String startDate, + @Parameter(in = ParameterIn.QUERY, description = "end date") @Valid String endDate, + @Parameter(in = ParameterIn.QUERY, description = "bounding box") @Valid String bbox + ){ + // test return + return ResponseEntity.ok().build(); + } + +} From 4f1bb34feb40f7c240b7503dbdcd5e5f3b3247ae Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Fri, 17 Jan 2025 16:04:25 +1100 Subject: [PATCH 03/23] add aws batch dependencies --- server/pom.xml | 10 +++++ .../server/core/configuration/AwsConfig.java | 18 +++++++++ .../server/core/service/AWSBatchService.java | 33 ++++++++++++++++ .../aodn/ogcapi/server/processes/RestApi.java | 38 +++++++++++++++++-- 4 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java diff --git a/server/pom.xml b/server/pom.xml index ad67d9e5..d1197dab 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -20,6 +20,16 @@ + + software.amazon.awssdk + batch + 2.29.52 + + + software.amazon.awssdk + auth + 2.29.52 + org.springframework.boot spring-boot-starter diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java new file mode 100644 index 00000000..a8862c2e --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java @@ -0,0 +1,18 @@ +package au.org.aodn.ogcapi.server.core.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; +import software.amazon.awssdk.services.batch.BatchClient; + +@Configuration +public class AwsConfig { + + @Bean + public BatchClient batchClient() { + return BatchClient + .builder() + .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) + .build(); + } +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java new file mode 100644 index 00000000..5a8f8551 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java @@ -0,0 +1,33 @@ +package au.org.aodn.ogcapi.server.core.service; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import software.amazon.awssdk.services.batch.BatchClient; +import software.amazon.awssdk.services.batch.model.SubmitJobRequest; +import software.amazon.awssdk.services.batch.model.SubmitJobResponse; + +import java.util.Map; + +@Service("AWSBatchService") +@Slf4j +public class AWSBatchService { + + private final BatchClient batchClient; + + public AWSBatchService(BatchClient batchClient) { + this.batchClient = batchClient; + } + + public String submitJob(String jobName, String jobQueue, String jobDefinition, Map parameters) { + SubmitJobRequest request = SubmitJobRequest.builder() + .jobName(jobName) + .jobQueue(jobQueue) + .jobDefinition(jobDefinition) + .parameters(parameters) + .build(); + + SubmitJobResponse response = batchClient.submitJob(request); + return response.jobId(); + } + +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java index 6d5c3d25..e3c170a4 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java @@ -1,28 +1,58 @@ package au.org.aodn.ogcapi.server.processes; +import au.org.aodn.ogcapi.server.core.service.AWSBatchService; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; import jakarta.validation.Valid; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import java.util.HashMap; +import java.util.Map; + @RestController("ProcessesRestApi") -@RequestMapping(value = "/api/v1/ogc") +@RequestMapping(value = "/api/v1/ogc/processes") public class RestApi { - @PostMapping(path="/processes/download/{collectionId}") + @Autowired + private AWSBatchService awsBatchService; + + @PostMapping(path="/download/{collectionId}") public ResponseEntity downloadData( @PathVariable("collectionId") String collectionId, @Parameter(in = ParameterIn.QUERY, description = "start date") @Valid String startDate, @Parameter(in = ParameterIn.QUERY, description = "end date") @Valid String endDate, @Parameter(in = ParameterIn.QUERY, description = "bounding box") @Valid String bbox ){ - // test return - return ResponseEntity.ok().build(); + try { + + System.out.println("AWS_ACCESS_KEY_ID: " + System.getenv("AWS_ACCESS_KEY_ID")); + System.out.println("AWS_SECRET_ACCESS_KEY: " + System.getenv("AWS_SECRET_ACCESS_KEY")); + + Map parameters = new HashMap<>(); + parameters.put("collectionId", collectionId); + parameters.put("startDate", startDate); + parameters.put("endDate", endDate); + parameters.put("bbox", bbox); + String jobId = awsBatchService.submitJob( + "test-downloading-job", + "test-downloading-job-queue", + "test-download-job-definition", + parameters); + + return ResponseEntity.ok("Job submitted with ID: " + jobId); + } catch (Exception e) { + System.out.println("Error while getting dataset"); + System.out.println(e.getMessage()); + return ResponseEntity.badRequest().body("Invalid parameters"); + } + + } } From cdd87f148f718caac1db347195a6422edc8b8f27 Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Tue, 21 Jan 2025 14:47:36 +1100 Subject: [PATCH 04/23] use environment variables to instead parameters --- .../server/core/service/AWSBatchService.java | 19 ++++++++++++++++--- .../aodn/ogcapi/server/processes/RestApi.java | 8 ++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java index 5a8f8551..22a8bd09 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java @@ -3,9 +3,11 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import software.amazon.awssdk.services.batch.BatchClient; +import software.amazon.awssdk.services.batch.model.KeyValuePair; import software.amazon.awssdk.services.batch.model.SubmitJobRequest; import software.amazon.awssdk.services.batch.model.SubmitJobResponse; +import java.util.List; import java.util.Map; @Service("AWSBatchService") @@ -19,15 +21,26 @@ public AWSBatchService(BatchClient batchClient) { } public String submitJob(String jobName, String jobQueue, String jobDefinition, Map parameters) { - SubmitJobRequest request = SubmitJobRequest.builder() + List environmentVariables = parameters.entrySet().stream() + .map(entry -> + KeyValuePair + .builder() + .name(entry.getKey()) + .value(entry.getValue()) + .build() + ) + .toList(); + + SubmitJobRequest submitJobRequest = SubmitJobRequest.builder() .jobName(jobName) .jobQueue(jobQueue) .jobDefinition(jobDefinition) .parameters(parameters) + .containerOverrides(co -> co.environment(environmentVariables)) .build(); - SubmitJobResponse response = batchClient.submitJob(request); - return response.jobId(); + SubmitJobResponse submitJobResponse = batchClient.submitJob(submitJobRequest); + return submitJobResponse.jobId(); } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java index e3c170a4..0888b3dd 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java @@ -35,10 +35,10 @@ public ResponseEntity downloadData( System.out.println("AWS_SECRET_ACCESS_KEY: " + System.getenv("AWS_SECRET_ACCESS_KEY")); Map parameters = new HashMap<>(); - parameters.put("collectionId", collectionId); - parameters.put("startDate", startDate); - parameters.put("endDate", endDate); - parameters.put("bbox", bbox); + parameters.put("COLLECTION_ID", collectionId); + parameters.put("START_DATE", startDate); + parameters.put("END_DATE", endDate); + parameters.put("BBOX", bbox); String jobId = awsBatchService.submitJob( "test-downloading-job", "test-downloading-job-queue", From 8bce5e116cdd024d6a9514b3eacfd9cd9f5a8cbc Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Wed, 29 Jan 2025 15:57:11 +1100 Subject: [PATCH 05/23] add enums for downloading data --- .../enumeration/DatasetDownloadEnums.java | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java new file mode 100644 index 00000000..6ee4b25a --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java @@ -0,0 +1,54 @@ +package au.org.aodn.ogcapi.server.core.model.enumeration; + +import lombok.Getter; + +/** + * the values are used for aws batch's environment variables + */ +public enum DatasetDownloadEnums { + + @Getter + public enum Condition{ + UUID("UUID"), + START_DATE("START_DATE"), + END_DATE("END_DATE"), + MIN_LATITUDE("MIN_LAT"), + MAX_LATITUDE("MAX_LAT"), + MIN_LONGITUDE("MIN_LON"), + MAX_LONGITUDE("MAX_LON"), + RECIPIENT("RECIPIENT") + ; + + private final String value; + + Condition(String value) { + this.value = value; + } + } + + @Getter + public enum JobDefinition{ + GENERATE_CSV_DATA_FILE("generate-csv-data-file"), + ; + + private final String value; + + JobDefinition(String value) { + this.value = value; + } + } + + @Getter + public enum JobQueue{ + GENERATING_CSV_DATA_FILE("generating-csv-data-file"), + ; + + private final String value; + + JobQueue(String value) { + this.value = value; + } + } + + +} From e3f5c7e79e3bb46d592f967c7ba770a9420ed700 Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Wed, 29 Jan 2025 15:58:39 +1100 Subject: [PATCH 06/23] correct an error in the enum --- .../enumeration/DatasetDownloadEnums.java | 99 +++++++++---------- 1 file changed, 47 insertions(+), 52 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java index 6ee4b25a..66f76bee 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java @@ -1,54 +1,49 @@ package au.org.aodn.ogcapi.server.core.model.enumeration; -import lombok.Getter; - -/** - * the values are used for aws batch's environment variables - */ -public enum DatasetDownloadEnums { - - @Getter - public enum Condition{ - UUID("UUID"), - START_DATE("START_DATE"), - END_DATE("END_DATE"), - MIN_LATITUDE("MIN_LAT"), - MAX_LATITUDE("MAX_LAT"), - MIN_LONGITUDE("MIN_LON"), - MAX_LONGITUDE("MAX_LON"), - RECIPIENT("RECIPIENT") - ; - - private final String value; - - Condition(String value) { - this.value = value; - } - } - - @Getter - public enum JobDefinition{ - GENERATE_CSV_DATA_FILE("generate-csv-data-file"), - ; - - private final String value; - - JobDefinition(String value) { - this.value = value; - } - } - - @Getter - public enum JobQueue{ - GENERATING_CSV_DATA_FILE("generating-csv-data-file"), - ; - - private final String value; - - JobQueue(String value) { - this.value = value; - } - } - - -} + import lombok.Getter; + + /** + * the values are used for aws batch's environment variables + */ + public class DatasetDownloadEnums { + + @Getter + public enum Condition { + UUID("UUID"), + START_DATE("START_DATE"), + END_DATE("END_DATE"), + MIN_LATITUDE("MIN_LAT"), + MAX_LATITUDE("MAX_LAT"), + MIN_LONGITUDE("MIN_LON"), + MAX_LONGITUDE("MAX_LON"), + RECIPIENT("RECIPIENT"); + + private final String value; + + Condition(String value) { + this.value = value; + } + } + + @Getter + public enum JobDefinition { + GENERATE_CSV_DATA_FILE("generate-csv-data-file"); + + private final String value; + + JobDefinition(String value) { + this.value = value; + } + } + + @Getter + public enum JobQueue { + GENERATING_CSV_DATA_FILE("generating-csv-data-file"); + + private final String value; + + JobQueue(String value) { + this.value = value; + } + } + } \ No newline at end of file From fa98f7c194a189d386e11ddc2aa4f5cdbd109fa2 Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Wed, 29 Jan 2025 17:41:09 +1100 Subject: [PATCH 07/23] update aws batch service --- .../server/core/service/AWSBatchService.java | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java index 22a8bd09..98283942 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java @@ -1,12 +1,13 @@ package au.org.aodn.ogcapi.server.core.service; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import software.amazon.awssdk.services.batch.BatchClient; -import software.amazon.awssdk.services.batch.model.KeyValuePair; -import software.amazon.awssdk.services.batch.model.SubmitJobRequest; -import software.amazon.awssdk.services.batch.model.SubmitJobResponse; +import software.amazon.awssdk.services.batch.model.*; +import java.io.File; +import java.io.IOException; import java.util.List; import java.util.Map; @@ -15,9 +16,11 @@ public class AWSBatchService { private final BatchClient batchClient; + private final ObjectMapper objectMapper; public AWSBatchService(BatchClient batchClient) { this.batchClient = batchClient; + this.objectMapper = new ObjectMapper(); } public String submitJob(String jobName, String jobQueue, String jobDefinition, Map parameters) { @@ -43,4 +46,35 @@ public String submitJob(String jobName, String jobQueue, String jobDefinition, M return submitJobResponse.jobId(); } + public JobQueueDetail getJobQueueBy(String name) { + var request = DescribeJobQueuesRequest + .builder() + .jobQueues(name) + .build(); + + var jobQueues = batchClient.describeJobQueues(request).jobQueues(); + + if (jobQueues != null && jobQueues.size() == 1) { + return jobQueues.get(0); + } + return null; + } + + public boolean isJobQueueValid(String jobQueueName) throws IOException { + + var remoteJobQueue = getJobQueueBy(jobQueueName); + var remoteJobQueueConfig = objectMapper.writeValueAsString(remoteJobQueue); + var localJobQueueConfig = getLocalJobQueueConfigBy(jobQueueName); + + return remoteJobQueueConfig.equals(localJobQueueConfig); + } + + public String getLocalJobQueueConfigBy(String jobQueueName) throws IOException { + // TODO: implement the path later + var configJsonPath = ""; + var jsonFile = new File(configJsonPath); + + return objectMapper.writeValueAsString(objectMapper.readValue(jsonFile, Map.class)); + } + } From a8d9641008fbfbb0192470b7c12adab7bdef3b66 Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Wed, 29 Jan 2025 17:54:19 +1100 Subject: [PATCH 08/23] add test cases --- .../server/core/service/AWSBatchService.java | 3 +- .../server/common/AWSBatchServiceTest.java | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java index 98283942..5919f7b4 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java @@ -60,6 +60,7 @@ public JobQueueDetail getJobQueueBy(String name) { return null; } + // TODO: implement the method later public boolean isJobQueueValid(String jobQueueName) throws IOException { var remoteJobQueue = getJobQueueBy(jobQueueName); @@ -69,8 +70,8 @@ public boolean isJobQueueValid(String jobQueueName) throws IOException { return remoteJobQueueConfig.equals(localJobQueueConfig); } + // TODO: implement the path later public String getLocalJobQueueConfigBy(String jobQueueName) throws IOException { - // TODO: implement the path later var configJsonPath = ""; var jsonFile = new File(configJsonPath); diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java new file mode 100644 index 00000000..cc380fcc --- /dev/null +++ b/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java @@ -0,0 +1,44 @@ +package au.org.aodn.ogcapi.server.common; + +import au.org.aodn.ogcapi.server.core.service.AWSBatchService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import software.amazon.awssdk.services.batch.BatchClient; +import software.amazon.awssdk.services.batch.model.*; + +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +public class AWSBatchServiceTest { + + @Mock + private BatchClient batchClient; + + @InjectMocks + private AWSBatchService awsBatchService; + + @BeforeEach + public void setUp() { + MockitoAnnotations.openMocks(this); + } + + @Test + public void testSubmitJob() { + // Arrange + String jobId = "test-job-id"; + SubmitJobResponse submitJobResponse = SubmitJobResponse.builder().jobId(jobId).build(); + when(batchClient.submitJob(any(SubmitJobRequest.class))).thenReturn(submitJobResponse); + + // Act + String result = awsBatchService.submitJob("test-job", "test-queue", "test-job-def", Collections.singletonMap("param1", "value1")); + + // Assert + assertEquals(jobId, result); + } +} \ No newline at end of file From 16504a2acaf214f2bd5de17b2ce08fb33ecae416 Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Wed, 29 Jan 2025 17:59:51 +1100 Subject: [PATCH 09/23] add test cases for process rest api --- .../aodn/ogcapi/server/processes/RestApi.java | 87 +++++++++++++------ .../ogcapi/server/processes/RestApiTest.java | 68 +++++++++++++++ 2 files changed, 128 insertions(+), 27 deletions(-) create mode 100644 server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java index 0888b3dd..2c964114 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java @@ -1,58 +1,91 @@ package au.org.aodn.ogcapi.server.processes; +import au.org.aodn.ogcapi.server.core.model.enumeration.DatasetDownloadEnums; import au.org.aodn.ogcapi.server.core.service.AWSBatchService; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; import jakarta.validation.Valid; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import java.util.HashMap; import java.util.Map; +@Slf4j @RestController("ProcessesRestApi") @RequestMapping(value = "/api/v1/ogc/processes") -public class RestApi { +public class RestApi { @Autowired private AWSBatchService awsBatchService; - @PostMapping(path="/download/{collectionId}") - public ResponseEntity downloadData( + @PostMapping(path = "/download/{collectionId}") + public ResponseEntity downloadData( @PathVariable("collectionId") String collectionId, - @Parameter(in = ParameterIn.QUERY, description = "start date") @Valid String startDate, - @Parameter(in = ParameterIn.QUERY, description = "end date") @Valid String endDate, - @Parameter(in = ParameterIn.QUERY, description = "bounding box") @Valid String bbox - ){ - try { + @Parameter(in = ParameterIn.QUERY, description = "start date") + @Valid + @RequestParam(name = "start_date") + String startDate, + + @Parameter(in = ParameterIn.QUERY, description = "end date") + @Valid + @RequestParam(name = "end_date") + String endDate, + + @Parameter(in = ParameterIn.QUERY, description = "minimum latitude") + @Valid + @RequestParam(name = "min_lat") + String minLat, + + @Parameter(in = ParameterIn.QUERY, description = "minimum longitude") + @Valid + @RequestParam(name = "min_lon") + String minLon, - System.out.println("AWS_ACCESS_KEY_ID: " + System.getenv("AWS_ACCESS_KEY_ID")); - System.out.println("AWS_SECRET_ACCESS_KEY: " + System.getenv("AWS_SECRET_ACCESS_KEY")); + @Parameter(in = ParameterIn.QUERY, description = "maximum latitude") + @Valid + @RequestParam(name = "max_lat") + String maxLat, + + @Parameter(in = ParameterIn.QUERY, description = "maximum longitude") + @Valid + @RequestParam(name = "max_lon") + String maxLon, + + @Parameter(in = ParameterIn.QUERY, description = "recipient") + @Valid + @RequestParam(name = "recipient") + String recipient + ) { + try { Map parameters = new HashMap<>(); - parameters.put("COLLECTION_ID", collectionId); - parameters.put("START_DATE", startDate); - parameters.put("END_DATE", endDate); - parameters.put("BBOX", bbox); + parameters.put(DatasetDownloadEnums.Condition.UUID.getValue(), collectionId); + parameters.put(DatasetDownloadEnums.Condition.START_DATE.getValue(), startDate); + parameters.put(DatasetDownloadEnums.Condition.END_DATE.getValue(), endDate); + parameters.put(DatasetDownloadEnums.Condition.MIN_LATITUDE.getValue(), minLat); + parameters.put(DatasetDownloadEnums.Condition.MIN_LONGITUDE.getValue(), minLon); + parameters.put(DatasetDownloadEnums.Condition.MAX_LATITUDE.getValue(), maxLat); + parameters.put(DatasetDownloadEnums.Condition.MAX_LONGITUDE.getValue(), maxLon); + parameters.put(DatasetDownloadEnums.Condition.RECIPIENT.getValue(), recipient); + + String jobId = awsBatchService.submitJob( - "test-downloading-job", - "test-downloading-job-queue", - "test-download-job-definition", + "generating-data-file-for-" + recipient.replaceAll("[^a-zA-Z0-9-_]", "-"), + DatasetDownloadEnums.JobQueue.GENERATING_CSV_DATA_FILE.getValue(), + DatasetDownloadEnums.JobDefinition.GENERATE_CSV_DATA_FILE.getValue(), parameters); - + log.info("Job submitted with ID: " + jobId); return ResponseEntity.ok("Job submitted with ID: " + jobId); } catch (Exception e) { - System.out.println("Error while getting dataset"); - System.out.println(e.getMessage()); - return ResponseEntity.badRequest().body("Invalid parameters"); - } + log.error("Error while getting dataset"); + log.error(e.getMessage()); + return ResponseEntity.badRequest().body("Error while getting dataset"); + } } - } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java new file mode 100644 index 00000000..94c93dc5 --- /dev/null +++ b/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java @@ -0,0 +1,68 @@ +package au.org.aodn.ogcapi.server.processes; + +import au.org.aodn.ogcapi.server.core.model.enumeration.DatasetDownloadEnums; +import au.org.aodn.ogcapi.server.core.service.AWSBatchService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.ResponseEntity; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +public class RestApiTest { + + @Mock + private AWSBatchService awsBatchService; + + @InjectMocks + private RestApi restApi; + + private Map parameters; + + @BeforeEach + public void setUp() { + parameters = new HashMap<>(); + parameters.put(DatasetDownloadEnums.Condition.UUID.getValue(), "collectionId"); + parameters.put(DatasetDownloadEnums.Condition.START_DATE.getValue(), "2023-01-01"); + parameters.put(DatasetDownloadEnums.Condition.END_DATE.getValue(), "2023-12-31"); + parameters.put(DatasetDownloadEnums.Condition.MIN_LATITUDE.getValue(), "10.0"); + parameters.put(DatasetDownloadEnums.Condition.MIN_LONGITUDE.getValue(), "20.0"); + parameters.put(DatasetDownloadEnums.Condition.MAX_LATITUDE.getValue(), "30.0"); + parameters.put(DatasetDownloadEnums.Condition.MAX_LONGITUDE.getValue(), "40.0"); + parameters.put(DatasetDownloadEnums.Condition.RECIPIENT.getValue(), "test@example.com"); + } + + @Test + public void testDownloadData_Success() { + when(awsBatchService.submitJob(any(), any(), any(), any())).thenReturn("jobId"); + + ResponseEntity response = restApi.downloadData( + "collectionId", "2023-01-01", "2023-12-31", "10.0", "20.0", "30.0", "40.0", "test@example.com"); + + assertTrue( response.getStatusCode().is2xxSuccessful()); + assertEquals("Job submitted with ID: jobId", response.getBody()); + verify(awsBatchService, times(1)).submitJob(any(), any(), any(), any()); + } + + @Test + public void testDownloadData_Error() { + when(awsBatchService.submitJob(any(), any(), any(), any())).thenThrow(new RuntimeException("Error")); + + ResponseEntity response = restApi.downloadData( + "collectionId", "2023-01-01", "2023-12-31", "10.0", "20.0", "30.0", "40.0", "test@example.com"); + + assertTrue( response.getStatusCode().is4xxClientError()); + assertEquals("Error while getting dataset", response.getBody()); + verify(awsBatchService, times(1)).submitJob(any(), any(), any(), any()); + } +} \ No newline at end of file From 57729bc75e54f4ad67ac2de5bd724be4b670255c Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Thu, 30 Jan 2025 11:01:17 +1100 Subject: [PATCH 10/23] minor fix --- .../enumeration/DatasetDownloadEnums.java | 95 ++++++++++--------- .../server/core/service/AWSBatchService.java | 37 ++++---- .../config/generate-csv-data-file.json | 18 ++++ .../ogcapi/server/processes/RestApiTest.java | 2 +- 4 files changed, 86 insertions(+), 66 deletions(-) create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/processes/config/generate-csv-data-file.json diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java index 66f76bee..fb4921fb 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java @@ -1,49 +1,50 @@ package au.org.aodn.ogcapi.server.core.model.enumeration; - import lombok.Getter; - - /** - * the values are used for aws batch's environment variables - */ - public class DatasetDownloadEnums { - - @Getter - public enum Condition { - UUID("UUID"), - START_DATE("START_DATE"), - END_DATE("END_DATE"), - MIN_LATITUDE("MIN_LAT"), - MAX_LATITUDE("MAX_LAT"), - MIN_LONGITUDE("MIN_LON"), - MAX_LONGITUDE("MAX_LON"), - RECIPIENT("RECIPIENT"); - - private final String value; - - Condition(String value) { - this.value = value; - } - } - - @Getter - public enum JobDefinition { - GENERATE_CSV_DATA_FILE("generate-csv-data-file"); - - private final String value; - - JobDefinition(String value) { - this.value = value; - } - } - - @Getter - public enum JobQueue { - GENERATING_CSV_DATA_FILE("generating-csv-data-file"); - - private final String value; - - JobQueue(String value) { - this.value = value; - } - } - } \ No newline at end of file +import lombok.Getter; + +/** + * the values are used for aws batch's environment variables + */ +public class DatasetDownloadEnums { + + @Getter + public enum Condition { + UUID("UUID"), + START_DATE("START_DATE"), + END_DATE("END_DATE"), + MIN_LATITUDE("MIN_LAT"), + MAX_LATITUDE("MAX_LAT"), + MIN_LONGITUDE("MIN_LON"), + MAX_LONGITUDE("MAX_LON"), + RECIPIENT("RECIPIENT"); + + private final String value; + + Condition(String value) { + this.value = value; + } + } + + @Getter + public enum JobDefinition { + GENERATE_CSV_DATA_FILE("generate-csv-data-file"); + + private final String value; + + JobDefinition(String value) { + this.value = value; + } + } + + @Getter + public enum JobQueue { + GENERATING_CSV_DATA_FILE("generate-csv-data-file"); + + private final String value; + + JobQueue(String value) { + this.value = value; + } + } +} + diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java index 5919f7b4..26b10732 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java @@ -18,12 +18,13 @@ public class AWSBatchService { private final BatchClient batchClient; private final ObjectMapper objectMapper; - public AWSBatchService(BatchClient batchClient) { + public AWSBatchService(BatchClient batchClient, ObjectMapper objectMapper) { this.batchClient = batchClient; - this.objectMapper = new ObjectMapper(); + this.objectMapper = objectMapper; } public String submitJob(String jobName, String jobQueue, String jobDefinition, Map parameters) { + List environmentVariables = parameters.entrySet().stream() .map(entry -> KeyValuePair @@ -46,7 +47,18 @@ public String submitJob(String jobName, String jobQueue, String jobDefinition, M return submitJobResponse.jobId(); } - public JobQueueDetail getJobQueueBy(String name) { + + + // TODO: This feature doesn't work yet. Will be implemented in the future as this one is not urgent + public boolean isJobQueueValid(String jobQueueName) throws IOException { + + var remoteJobQueueDetail = getRemoteJobQueueBy(jobQueueName); + var localJobQueueDetail = getLocalJobQueueDetailBy(jobQueueName); + + return remoteJobQueueDetail.equals(localJobQueueDetail); + } + + public JobQueueDetail getRemoteJobQueueBy(String name) { var request = DescribeJobQueuesRequest .builder() .jobQueues(name) @@ -60,22 +72,11 @@ public JobQueueDetail getJobQueueBy(String name) { return null; } - // TODO: implement the method later - public boolean isJobQueueValid(String jobQueueName) throws IOException { - - var remoteJobQueue = getJobQueueBy(jobQueueName); - var remoteJobQueueConfig = objectMapper.writeValueAsString(remoteJobQueue); - var localJobQueueConfig = getLocalJobQueueConfigBy(jobQueueName); - - return remoteJobQueueConfig.equals(localJobQueueConfig); - } - - // TODO: implement the path later - public String getLocalJobQueueConfigBy(String jobQueueName) throws IOException { - var configJsonPath = ""; + public JobQueueDetail getLocalJobQueueDetailBy(String jobQueueName) throws IOException { + var configJsonPath = "server/src/main/java/au/org/aodn/ogcapi/server/processes/config/" + jobQueueName + ".json"; var jsonFile = new File(configJsonPath); - - return objectMapper.writeValueAsString(objectMapper.readValue(jsonFile, Map.class)); + var jsonStr = objectMapper.writeValueAsString(objectMapper.readValue(jsonFile, Object.class)); + return objectMapper.readValue(jsonStr, JobQueueDetail.class); } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/config/generate-csv-data-file.json b/server/src/main/java/au/org/aodn/ogcapi/server/processes/config/generate-csv-data-file.json new file mode 100644 index 00000000..e7ffadfb --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/config/generate-csv-data-file.json @@ -0,0 +1,18 @@ +{ + "jobQueueName": "generate-csv-data-file", + "jobQueueArn": "arn:aws:batch:ap-southeast-2:704910415367:job-queue/generate-csv-data-file", + "state": "ENABLED", + "status": "VALID", + "statusReason": "JobQueue Healthy", + "priority": 1, + "computeEnvironmentOrder": [ + { + "order": 1, + "computeEnvironment": "arn:aws:batch:ap-southeast-2:704910415367:compute-environment/generate-csv-data-file" + } + ], + "serviceEnvironmentOrder": [], + "jobQueueType": "ECS_FARGATE", + "tags": {}, + "jobStateTimeLimitActions": [] +} diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java index 94c93dc5..bf9eaa27 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java @@ -65,4 +65,4 @@ public void testDownloadData_Error() { assertEquals("Error while getting dataset", response.getBody()); verify(awsBatchService, times(1)).submitJob(any(), any(), any(), any()); } -} \ No newline at end of file +} From 2a24741ff5e9797c9a2bc4aa9c024a74e8d6fb3a Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Thu, 30 Jan 2025 11:41:01 +1100 Subject: [PATCH 11/23] fix for failure test about aws --- .../au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java index a8862c2e..0fc3dc6e 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java @@ -3,6 +3,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; +import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.batch.BatchClient; @Configuration @@ -12,6 +13,7 @@ public class AwsConfig { public BatchClient batchClient() { return BatchClient .builder() + .region(Region.AP_SOUTHEAST_2) .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) .build(); } From d832709b783eab54331700afc41da3d4c866626c Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Thu, 30 Jan 2025 11:52:11 +1100 Subject: [PATCH 12/23] pre-commit fix --- .../au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java index cc380fcc..dc20beb3 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java @@ -41,4 +41,4 @@ public void testSubmitJob() { // Assert assertEquals(jobId, result); } -} \ No newline at end of file +} From a5cc368e7893eeacf6551b524c2eb254de3a0a9c Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Thu, 30 Jan 2025 13:50:39 +1100 Subject: [PATCH 13/23] pre-commit fix --- .../server/core/model/enumeration/DatasetDownloadEnums.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java index fb4921fb..81a9f497 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java @@ -47,4 +47,3 @@ public enum JobQueue { } } } - From f5dd3524a30def375790049395cf25c8e08ba430 Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Thu, 30 Jan 2025 16:57:00 +1100 Subject: [PATCH 14/23] minor fixing --- .../ogcapi/server/core/configuration/AwsConfig.java | 13 ++++++++++++- .../ogcapi/server/core/service/AWSBatchService.java | 2 -- server/src/main/resources/application.yaml | 3 +++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java index 0fc3dc6e..92462d73 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java @@ -1,5 +1,8 @@ package au.org.aodn.ogcapi.server.core.configuration; +import au.org.aodn.ogcapi.server.core.service.AWSBatchService; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; @@ -9,12 +12,20 @@ @Configuration public class AwsConfig { + @Value("${aws.region}") + private String awsRegion; + @Bean public BatchClient batchClient() { return BatchClient .builder() - .region(Region.AP_SOUTHEAST_2) + .region(Region.of(awsRegion)) .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) .build(); } + + @Bean + public AWSBatchService awsBatchService(BatchClient batchClient, ObjectMapper objectMapper) { + return new AWSBatchService(batchClient, objectMapper); + } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java index 26b10732..01cb8252 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; import software.amazon.awssdk.services.batch.BatchClient; import software.amazon.awssdk.services.batch.model.*; @@ -11,7 +10,6 @@ import java.util.List; import java.util.Map; -@Service("AWSBatchService") @Slf4j public class AWSBatchService { diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml index 7c866e21..07470c4c 100644 --- a/server/src/main/resources/application.yaml +++ b/server/src/main/resources/application.yaml @@ -19,6 +19,9 @@ elasticsearch: path: search_suggestions fields: abstract_phrases, parameter_vocabs_sayt, platform_vocabs_sayt, organisation_vocabs_sayt +aws: + region: ap-southeast-2 + api: host: http://localhost:${server.port} From 29f8e4f2f7d74f5ab4baa4969f098a8011f46c22 Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Fri, 31 Jan 2025 10:41:09 +1100 Subject: [PATCH 15/23] make process api to implement necessary interface --- server/pom.xml | 5 +++++ .../aodn/ogcapi/server/processes/RestApi.java | 22 ++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/server/pom.xml b/server/pom.xml index d1197dab..8520de2f 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -81,6 +81,11 @@ ogcapi-features-java ${project.parent.version} + + au.org.aodn.ogcapi + ogcapi-processes-java + ${project.parent.version} + co.elastic.clients elasticsearch-java diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java index 2c964114..bda749fc 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java @@ -1,6 +1,10 @@ package au.org.aodn.ogcapi.server.processes; +import au.org.aodn.ogcapi.processes.api.ProcessesApi; +import au.org.aodn.ogcapi.processes.model.Execute; +import au.org.aodn.ogcapi.processes.model.InlineResponse200; +import au.org.aodn.ogcapi.processes.model.ProcessList; import au.org.aodn.ogcapi.server.core.model.enumeration.DatasetDownloadEnums; import au.org.aodn.ogcapi.server.core.service.AWSBatchService; import io.swagger.v3.oas.annotations.Parameter; @@ -8,6 +12,7 @@ import jakarta.validation.Valid; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; @@ -17,7 +22,7 @@ @Slf4j @RestController("ProcessesRestApi") @RequestMapping(value = "/api/v1/ogc/processes") -public class RestApi { +public class RestApi implements ProcessesApi { @Autowired private AWSBatchService awsBatchService; @@ -88,4 +93,19 @@ public ResponseEntity downloadData( return ResponseEntity.badRequest().body("Error while getting dataset"); } } + + @Override + public ResponseEntity execute(String processID, Execute body) { + return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build(); + } + + @Override + public ResponseEntity getProcessDescription(String processID) { + return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build(); + } + + @Override + public ResponseEntity getProcesses() { + return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build(); + } } From 7b9d0a3b2a31f13381c5920f1615020f70bd6ce2 Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Fri, 31 Jan 2025 13:35:06 +1100 Subject: [PATCH 16/23] test fixing --- .../org/aodn/ogcapi/server/processes/RestApi.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java index bda749fc..24652bfa 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java @@ -9,6 +9,7 @@ import au.org.aodn.ogcapi.server.core.service.AWSBatchService; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.Valid; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -95,10 +96,21 @@ public ResponseEntity downloadData( } @Override - public ResponseEntity execute(String processID, Execute body) { + @RequestMapping(value = "/processes/{processID}/execution", + produces = { "application/json", "text/html" }, + consumes = { "application/json" }, + method = RequestMethod.POST) + public ResponseEntity execute( + @Parameter(in = ParameterIn.PATH, required=true, schema=@Schema()) + @PathVariable("processID") + String processID, + @Parameter(in = ParameterIn.DEFAULT, description = "Mandatory execute request JSON", required=true, schema=@Schema()) + @Valid + @RequestBody Execute body){ return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build(); } + @Override public ResponseEntity getProcessDescription(String processID) { return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build(); From 6ebec88204f1970255ec2b847224758ba1076af2 Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Fri, 31 Jan 2025 15:26:47 +1100 Subject: [PATCH 17/23] move api functions --- .../ogcapi/server/core/model/InlineValue.java | 5 ++ .../core/model/enumeration/ProcessIdEnum.java | 15 ++++ .../aodn/ogcapi/server/processes/RestApi.java | 74 ++++++++++--------- 3 files changed, 61 insertions(+), 33 deletions(-) create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/model/InlineValue.java create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/ProcessIdEnum.java diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/InlineValue.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/InlineValue.java new file mode 100644 index 00000000..bfeeb12c --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/InlineValue.java @@ -0,0 +1,5 @@ +package au.org.aodn.ogcapi.server.core.model; + +import au.org.aodn.ogcapi.processes.model.InlineOrRefData; + +public record InlineValue(String message) implements InlineOrRefData {} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/ProcessIdEnum.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/ProcessIdEnum.java new file mode 100644 index 00000000..117b4c06 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/ProcessIdEnum.java @@ -0,0 +1,15 @@ +package au.org.aodn.ogcapi.server.core.model.enumeration; + +import lombok.Getter; + +@Getter +public enum ProcessIdEnum { + DOWNLOAD_DATASET("download"), + ; + + private final String value; + + ProcessIdEnum(String value) { + this.value = value; + } +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java index 24652bfa..91020283 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java @@ -5,7 +5,10 @@ import au.org.aodn.ogcapi.processes.model.Execute; import au.org.aodn.ogcapi.processes.model.InlineResponse200; import au.org.aodn.ogcapi.processes.model.ProcessList; +import au.org.aodn.ogcapi.processes.model.Results; +import au.org.aodn.ogcapi.server.core.model.InlineValue; import au.org.aodn.ogcapi.server.core.model.enumeration.DatasetDownloadEnums; +import au.org.aodn.ogcapi.server.core.model.enumeration.ProcessIdEnum; import au.org.aodn.ogcapi.server.core.service.AWSBatchService; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; @@ -22,54 +25,26 @@ @Slf4j @RestController("ProcessesRestApi") -@RequestMapping(value = "/api/v1/ogc/processes") +@RequestMapping(value = "/api/v1/ogc") public class RestApi implements ProcessesApi { @Autowired private AWSBatchService awsBatchService; - @PostMapping(path = "/download/{collectionId}") public ResponseEntity downloadData( - @PathVariable("collectionId") String collectionId, - @Parameter(in = ParameterIn.QUERY, description = "start date") - @Valid - @RequestParam(name = "start_date") + String id, String startDate, - - @Parameter(in = ParameterIn.QUERY, description = "end date") - @Valid - @RequestParam(name = "end_date") String endDate, - - @Parameter(in = ParameterIn.QUERY, description = "minimum latitude") - @Valid - @RequestParam(name = "min_lat") String minLat, - - @Parameter(in = ParameterIn.QUERY, description = "minimum longitude") - @Valid - @RequestParam(name = "min_lon") String minLon, - - @Parameter(in = ParameterIn.QUERY, description = "maximum latitude") - @Valid - @RequestParam(name = "max_lat") String maxLat, - - @Parameter(in = ParameterIn.QUERY, description = "maximum longitude") - @Valid - @RequestParam(name = "max_lon") String maxLon, - - @Parameter(in = ParameterIn.QUERY, description = "recipient") - @Valid - @RequestParam(name = "recipient") String recipient ) { try { Map parameters = new HashMap<>(); - parameters.put(DatasetDownloadEnums.Condition.UUID.getValue(), collectionId); + parameters.put(DatasetDownloadEnums.Condition.UUID.getValue(), id); parameters.put(DatasetDownloadEnums.Condition.START_DATE.getValue(), startDate); parameters.put(DatasetDownloadEnums.Condition.END_DATE.getValue(), endDate); parameters.put(DatasetDownloadEnums.Condition.MIN_LATITUDE.getValue(), minLat); @@ -88,7 +63,6 @@ public ResponseEntity downloadData( return ResponseEntity.ok("Job submitted with ID: " + jobId); } catch (Exception e) { - log.error("Error while getting dataset"); log.error(e.getMessage()); return ResponseEntity.badRequest().body("Error while getting dataset"); @@ -107,7 +81,41 @@ public ResponseEntity execute( @Parameter(in = ParameterIn.DEFAULT, description = "Mandatory execute request JSON", required=true, schema=@Schema()) @Valid @RequestBody Execute body){ - return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build(); + + if (processID.equals(ProcessIdEnum.DOWNLOAD_DATASET.getValue())) { + try { + var response = downloadData( + (String) body.getInputs().get("collectionId"), + (String) body.getInputs().get("start_date"), + (String) body.getInputs().get("end_date"), + (String) body.getInputs().get("min_lat"), + (String) body.getInputs().get("min_lon"), + (String) body.getInputs().get("max_lat"), + (String) body.getInputs().get("max_lon"), + (String) body.getInputs().get("recipient") + ); + + var value = new InlineValue(response.getBody()); + var results = new Results(); + results.put("message", value); + + return ResponseEntity.ok(results); + + } catch (Exception e) { + log.error(e.getMessage()); + var response = new Results(); + var value = new InlineValue("Error while getting dataset"); + response.put("error", value); + + return ResponseEntity.badRequest().body(response); + } + } + + var response = new Results(); + var value = new InlineValue("Unknown process ID: " + processID); + response.put("error", value); + + return ResponseEntity.badRequest().body(response); } From 2b04d6651d2a29969cab8d14cffe4d54a7ee25d1 Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Fri, 31 Jan 2025 15:27:15 +1100 Subject: [PATCH 18/23] change scope of downloaddata function --- .../main/java/au/org/aodn/ogcapi/server/processes/RestApi.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java index 91020283..55a56d6e 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java @@ -31,7 +31,7 @@ public class RestApi implements ProcessesApi { @Autowired private AWSBatchService awsBatchService; - public ResponseEntity downloadData( + private ResponseEntity downloadData( String id, String startDate, String endDate, From c9833419002566fe0307efc2e6ff37e965d865ab Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Fri, 31 Jan 2025 15:36:19 +1100 Subject: [PATCH 19/23] fix test cases --- .../ogcapi/server/processes/RestApiTest.java | 58 +++++++++++-------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java index bf9eaa27..6d551e75 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java @@ -1,6 +1,10 @@ package au.org.aodn.ogcapi.server.processes; -import au.org.aodn.ogcapi.server.core.model.enumeration.DatasetDownloadEnums; +import au.org.aodn.ogcapi.processes.model.Execute; +import au.org.aodn.ogcapi.processes.model.InlineResponse200; +import au.org.aodn.ogcapi.processes.model.Results; +import au.org.aodn.ogcapi.server.core.model.InlineValue; +import au.org.aodn.ogcapi.server.core.model.enumeration.ProcessIdEnum; import au.org.aodn.ogcapi.server.core.service.AWSBatchService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -27,42 +31,46 @@ public class RestApiTest { @InjectMocks private RestApi restApi; - private Map parameters; + private Execute executeRequest; @BeforeEach public void setUp() { - parameters = new HashMap<>(); - parameters.put(DatasetDownloadEnums.Condition.UUID.getValue(), "collectionId"); - parameters.put(DatasetDownloadEnums.Condition.START_DATE.getValue(), "2023-01-01"); - parameters.put(DatasetDownloadEnums.Condition.END_DATE.getValue(), "2023-12-31"); - parameters.put(DatasetDownloadEnums.Condition.MIN_LATITUDE.getValue(), "10.0"); - parameters.put(DatasetDownloadEnums.Condition.MIN_LONGITUDE.getValue(), "20.0"); - parameters.put(DatasetDownloadEnums.Condition.MAX_LATITUDE.getValue(), "30.0"); - parameters.put(DatasetDownloadEnums.Condition.MAX_LONGITUDE.getValue(), "40.0"); - parameters.put(DatasetDownloadEnums.Condition.RECIPIENT.getValue(), "test@example.com"); + Map inputs = new HashMap<>(); + inputs.put("collectionId", "collectionId"); + inputs.put("start_date", "2023-01-01"); + inputs.put("end_date", "2023-12-31"); + inputs.put("min_lat", "10.0"); + inputs.put("min_lon", "20.0"); + inputs.put("max_lat", "30.0"); + inputs.put("max_lon", "40.0"); + inputs.put("recipient", "test@example.com"); + + executeRequest = new Execute(); + executeRequest.setInputs(inputs); } @Test - public void testDownloadData_Success() { + public void testExecute() { when(awsBatchService.submitJob(any(), any(), any(), any())).thenReturn("jobId"); - ResponseEntity response = restApi.downloadData( - "collectionId", "2023-01-01", "2023-12-31", "10.0", "20.0", "30.0", "40.0", "test@example.com"); + ResponseEntity response = restApi.execute(ProcessIdEnum.DOWNLOAD_DATASET.getValue(), executeRequest); - assertTrue( response.getStatusCode().is2xxSuccessful()); - assertEquals("Job submitted with ID: jobId", response.getBody()); + assertTrue(response.getStatusCode().is2xxSuccessful()); + Results results = (Results) response.getBody(); + assert results != null; + InlineValue message = (InlineValue) results.get("message"); + assertEquals("Job submitted with ID: jobId", message.message()); verify(awsBatchService, times(1)).submitJob(any(), any(), any(), any()); } @Test - public void testDownloadData_Error() { - when(awsBatchService.submitJob(any(), any(), any(), any())).thenThrow(new RuntimeException("Error")); - - ResponseEntity response = restApi.downloadData( - "collectionId", "2023-01-01", "2023-12-31", "10.0", "20.0", "30.0", "40.0", "test@example.com"); + public void testExecute_UnknownProcessId() { + ResponseEntity response = restApi.execute("unknownProcessId", executeRequest); - assertTrue( response.getStatusCode().is4xxClientError()); - assertEquals("Error while getting dataset", response.getBody()); - verify(awsBatchService, times(1)).submitJob(any(), any(), any(), any()); + assertTrue(response.getStatusCode().is4xxClientError()); + Results results = (Results) response.getBody(); + assert results != null; + InlineValue error = (InlineValue) results.get("error"); + assertEquals("Unknown process ID: unknownProcessId", error.message()); } -} +} \ No newline at end of file From b09b43561e68e50a98cdf7421b8aa77927e1dd0b Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Fri, 31 Jan 2025 15:37:46 +1100 Subject: [PATCH 20/23] fix pre-commit --- .../java/au/org/aodn/ogcapi/server/processes/RestApiTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java index 6d551e75..44bb8501 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java @@ -73,4 +73,4 @@ public void testExecute_UnknownProcessId() { InlineValue error = (InlineValue) results.get("error"); assertEquals("Unknown process ID: unknownProcessId", error.message()); } -} \ No newline at end of file +} From b3d2409de98624dd9d681ae37b1e0697619ca05c Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Fri, 31 Jan 2025 16:03:03 +1100 Subject: [PATCH 21/23] change functions location --- .../server/core/service/AWSBatchService.java | 43 ++++++++++++++- .../aodn/ogcapi/server/processes/RestApi.java | 46 +--------------- .../server/common/AWSBatchServiceTest.java | 35 +++++++++--- .../ogcapi/server/processes/RestApiTest.java | 55 +++++++++++-------- 4 files changed, 103 insertions(+), 76 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java index 01cb8252..6ffe2119 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java @@ -1,12 +1,15 @@ package au.org.aodn.ogcapi.server.core.service; +import au.org.aodn.ogcapi.server.core.model.enumeration.DatasetDownloadEnums; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; +import org.springframework.http.ResponseEntity; import software.amazon.awssdk.services.batch.BatchClient; import software.amazon.awssdk.services.batch.model.*; import java.io.File; import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -21,7 +24,45 @@ public AWSBatchService(BatchClient batchClient, ObjectMapper objectMapper) { this.objectMapper = objectMapper; } - public String submitJob(String jobName, String jobQueue, String jobDefinition, Map parameters) { + public ResponseEntity downloadData( + String id, + String startDate, + String endDate, + String minLat, + String minLon, + String maxLat, + String maxLon, + String recipient + ) { + try { + + Map parameters = new HashMap<>(); + parameters.put(DatasetDownloadEnums.Condition.UUID.getValue(), id); + parameters.put(DatasetDownloadEnums.Condition.START_DATE.getValue(), startDate); + parameters.put(DatasetDownloadEnums.Condition.END_DATE.getValue(), endDate); + parameters.put(DatasetDownloadEnums.Condition.MIN_LATITUDE.getValue(), minLat); + parameters.put(DatasetDownloadEnums.Condition.MIN_LONGITUDE.getValue(), minLon); + parameters.put(DatasetDownloadEnums.Condition.MAX_LATITUDE.getValue(), maxLat); + parameters.put(DatasetDownloadEnums.Condition.MAX_LONGITUDE.getValue(), maxLon); + parameters.put(DatasetDownloadEnums.Condition.RECIPIENT.getValue(), recipient); + + + String jobId = submitJob( + "generating-data-file-for-" + recipient.replaceAll("[^a-zA-Z0-9-_]", "-"), + DatasetDownloadEnums.JobQueue.GENERATING_CSV_DATA_FILE.getValue(), + DatasetDownloadEnums.JobDefinition.GENERATE_CSV_DATA_FILE.getValue(), + parameters); + log.info("Job submitted with ID: " + jobId); + return ResponseEntity.ok("Job submitted with ID: " + jobId); + } catch (Exception e) { + + log.error("Error while getting dataset"); + log.error(e.getMessage()); + return ResponseEntity.badRequest().body("Error while getting dataset"); + } + } + + private String submitJob(String jobName, String jobQueue, String jobDefinition, Map parameters) { List environmentVariables = parameters.entrySet().stream() .map(entry -> diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java index 55a56d6e..2291edaa 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java @@ -7,7 +7,6 @@ import au.org.aodn.ogcapi.processes.model.ProcessList; import au.org.aodn.ogcapi.processes.model.Results; import au.org.aodn.ogcapi.server.core.model.InlineValue; -import au.org.aodn.ogcapi.server.core.model.enumeration.DatasetDownloadEnums; import au.org.aodn.ogcapi.server.core.model.enumeration.ProcessIdEnum; import au.org.aodn.ogcapi.server.core.service.AWSBatchService; import io.swagger.v3.oas.annotations.Parameter; @@ -20,9 +19,6 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; -import java.util.HashMap; -import java.util.Map; - @Slf4j @RestController("ProcessesRestApi") @RequestMapping(value = "/api/v1/ogc") @@ -31,45 +27,9 @@ public class RestApi implements ProcessesApi { @Autowired private AWSBatchService awsBatchService; - private ResponseEntity downloadData( - String id, - String startDate, - String endDate, - String minLat, - String minLon, - String maxLat, - String maxLon, - String recipient - ) { - try { - - Map parameters = new HashMap<>(); - parameters.put(DatasetDownloadEnums.Condition.UUID.getValue(), id); - parameters.put(DatasetDownloadEnums.Condition.START_DATE.getValue(), startDate); - parameters.put(DatasetDownloadEnums.Condition.END_DATE.getValue(), endDate); - parameters.put(DatasetDownloadEnums.Condition.MIN_LATITUDE.getValue(), minLat); - parameters.put(DatasetDownloadEnums.Condition.MIN_LONGITUDE.getValue(), minLon); - parameters.put(DatasetDownloadEnums.Condition.MAX_LATITUDE.getValue(), maxLat); - parameters.put(DatasetDownloadEnums.Condition.MAX_LONGITUDE.getValue(), maxLon); - parameters.put(DatasetDownloadEnums.Condition.RECIPIENT.getValue(), recipient); - - - String jobId = awsBatchService.submitJob( - "generating-data-file-for-" + recipient.replaceAll("[^a-zA-Z0-9-_]", "-"), - DatasetDownloadEnums.JobQueue.GENERATING_CSV_DATA_FILE.getValue(), - DatasetDownloadEnums.JobDefinition.GENERATE_CSV_DATA_FILE.getValue(), - parameters); - log.info("Job submitted with ID: " + jobId); - return ResponseEntity.ok("Job submitted with ID: " + jobId); - } catch (Exception e) { - - log.error("Error while getting dataset"); - log.error(e.getMessage()); - return ResponseEntity.badRequest().body("Error while getting dataset"); - } - } - @Override + // because the produces value in the interface declaration includes "/_" which may + // cause exception thrown sometimes. So i re-declared the produces value here @RequestMapping(value = "/processes/{processID}/execution", produces = { "application/json", "text/html" }, consumes = { "application/json" }, @@ -84,7 +44,7 @@ public ResponseEntity execute( if (processID.equals(ProcessIdEnum.DOWNLOAD_DATASET.getValue())) { try { - var response = downloadData( + var response = awsBatchService.downloadData( (String) body.getInputs().get("collectionId"), (String) body.getInputs().get("start_date"), (String) body.getInputs().get("end_date"), diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java index dc20beb3..470435df 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java @@ -1,19 +1,20 @@ package au.org.aodn.ogcapi.server.common; import au.org.aodn.ogcapi.server.core.service.AWSBatchService; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import org.springframework.http.ResponseEntity; import software.amazon.awssdk.services.batch.BatchClient; -import software.amazon.awssdk.services.batch.model.*; - -import java.util.Collections; +import software.amazon.awssdk.services.batch.model.SubmitJobRequest; +import software.amazon.awssdk.services.batch.model.SubmitJobResponse; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; public class AWSBatchServiceTest { @@ -29,16 +30,32 @@ public void setUp() { } @Test - public void testSubmitJob() { + public void testDownloadDataSuccess() { // Arrange - String jobId = "test-job-id"; + String jobId = "12345"; SubmitJobResponse submitJobResponse = SubmitJobResponse.builder().jobId(jobId).build(); when(batchClient.submitJob(any(SubmitJobRequest.class))).thenReturn(submitJobResponse); // Act - String result = awsBatchService.submitJob("test-job", "test-queue", "test-job-def", Collections.singletonMap("param1", "value1")); + ResponseEntity response = awsBatchService.downloadData( + "id", "2021-01-01", "2021-01-31", "10.0", "20.0", "30.0", "40.0", "recipient@example.com"); + + // Assert + assertEquals(ResponseEntity.ok("Job submitted with ID: " + jobId), response); + verify(batchClient, times(1)).submitJob(any(SubmitJobRequest.class)); + } + + @Test + public void testDownloadDataFailure() { + // Arrange + when(batchClient.submitJob(any(SubmitJobRequest.class))).thenThrow(new RuntimeException("AWS Batch error")); + + // Act + ResponseEntity response = awsBatchService.downloadData( + "id", "2021-01-01", "2021-01-31", "10.0", "20.0", "30.0", "40.0", "recipient@example.com"); // Assert - assertEquals(jobId, result); + assertEquals(ResponseEntity.badRequest().body("Error while getting dataset"), response); + verify(batchClient, times(1)).submitJob(any(SubmitJobRequest.class)); } -} +} \ No newline at end of file diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java index 44bb8501..fc9c6b86 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java @@ -18,9 +18,8 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) public class RestApiTest { @@ -35,42 +34,52 @@ public class RestApiTest { @BeforeEach public void setUp() { + executeRequest = new Execute(); Map inputs = new HashMap<>(); - inputs.put("collectionId", "collectionId"); + inputs.put("collectionId", "test-collection-id"); inputs.put("start_date", "2023-01-01"); - inputs.put("end_date", "2023-12-31"); - inputs.put("min_lat", "10.0"); - inputs.put("min_lon", "20.0"); - inputs.put("max_lat", "30.0"); - inputs.put("max_lon", "40.0"); + inputs.put("end_date", "2023-01-31"); + inputs.put("min_lat", "-10.0"); + inputs.put("min_lon", "110.0"); + inputs.put("max_lat", "10.0"); + inputs.put("max_lon", "150.0"); inputs.put("recipient", "test@example.com"); - - executeRequest = new Execute(); executeRequest.setInputs(inputs); } @Test - public void testExecute() { - when(awsBatchService.submitJob(any(), any(), any(), any())).thenReturn("jobId"); + public void testExecuteDownloadDatasetSuccess() { + when(awsBatchService.downloadData(anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString())) + .thenReturn(ResponseEntity.ok("Job submitted with ID: test-job-id")); ResponseEntity response = restApi.execute(ProcessIdEnum.DOWNLOAD_DATASET.getValue(), executeRequest); - assertTrue(response.getStatusCode().is2xxSuccessful()); + assertEquals(200, response.getStatusCode().value()); Results results = (Results) response.getBody(); - assert results != null; InlineValue message = (InlineValue) results.get("message"); - assertEquals("Job submitted with ID: jobId", message.message()); - verify(awsBatchService, times(1)).submitJob(any(), any(), any(), any()); + assertEquals("Job submitted with ID: test-job-id", message.message()); + } + + @Test + public void testExecuteDownloadDatasetError() { + when(awsBatchService.downloadData(anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString())) + .thenThrow(new RuntimeException("Error while getting dataset")); + + ResponseEntity response = restApi.execute(ProcessIdEnum.DOWNLOAD_DATASET.getValue(), executeRequest); + + assertEquals(400, response.getStatusCode().value()); + Results results = (Results) response.getBody(); + InlineValue error = (InlineValue) results.get("error"); + assertEquals("Error while getting dataset", error.message()); } @Test - public void testExecute_UnknownProcessId() { - ResponseEntity response = restApi.execute("unknownProcessId", executeRequest); + public void testExecuteUnknownProcessId() { + ResponseEntity response = restApi.execute("unknown-process-id", executeRequest); - assertTrue(response.getStatusCode().is4xxClientError()); + assertEquals(400, response.getStatusCode().value()); Results results = (Results) response.getBody(); - assert results != null; InlineValue error = (InlineValue) results.get("error"); - assertEquals("Unknown process ID: unknownProcessId", error.message()); + assertEquals("Unknown process ID: unknown-process-id", error.message()); } -} +} \ No newline at end of file From b2045138225bf87e04616e1622e8a118f4088872 Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Fri, 31 Jan 2025 16:05:57 +1100 Subject: [PATCH 22/23] fix pre-commit errors --- .../org/aodn/ogcapi/server/common/AWSBatchServiceTest.java | 3 +-- .../au/org/aodn/ogcapi/server/processes/RestApiTest.java | 5 ++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java index 470435df..560522dd 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java @@ -1,7 +1,6 @@ package au.org.aodn.ogcapi.server.common; import au.org.aodn.ogcapi.server.core.service.AWSBatchService; -import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.InjectMocks; @@ -58,4 +57,4 @@ public void testDownloadDataFailure() { assertEquals(ResponseEntity.badRequest().body("Error while getting dataset"), response); verify(batchClient, times(1)).submitJob(any(SubmitJobRequest.class)); } -} \ No newline at end of file +} diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java index fc9c6b86..1fc8caad 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java @@ -56,6 +56,7 @@ public void testExecuteDownloadDatasetSuccess() { assertEquals(200, response.getStatusCode().value()); Results results = (Results) response.getBody(); + assert results != null; InlineValue message = (InlineValue) results.get("message"); assertEquals("Job submitted with ID: test-job-id", message.message()); } @@ -69,6 +70,7 @@ public void testExecuteDownloadDatasetError() { assertEquals(400, response.getStatusCode().value()); Results results = (Results) response.getBody(); + assert results != null; InlineValue error = (InlineValue) results.get("error"); assertEquals("Error while getting dataset", error.message()); } @@ -79,7 +81,8 @@ public void testExecuteUnknownProcessId() { assertEquals(400, response.getStatusCode().value()); Results results = (Results) response.getBody(); + assert results != null; InlineValue error = (InlineValue) results.get("error"); assertEquals("Unknown process ID: unknown-process-id", error.message()); } -} \ No newline at end of file +} From 015ec8d21ccf198845d691ddb5ebddf2bed2beab Mon Sep 17 00:00:00 2001 From: HuaizhiDai Date: Fri, 31 Jan 2025 17:27:32 +1100 Subject: [PATCH 23/23] relocate aws services --- .../ogcapi/server/core/configuration/AwsConfig.java | 6 +++--- .../au/org/aodn/ogcapi/server/processes/RestApi.java | 5 ++--- .../RestServices.java} | 6 +++--- .../org/aodn/ogcapi/server/processes/RestApiTest.java | 7 +++---- .../RestServicesTest.java} | 11 +++++------ 5 files changed, 16 insertions(+), 19 deletions(-) rename server/src/main/java/au/org/aodn/ogcapi/server/{core/service/AWSBatchService.java => processes/RestServices.java} (96%) rename server/src/test/java/au/org/aodn/ogcapi/server/{common/AWSBatchServiceTest.java => processes/RestServicesTest.java} (84%) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java index 92462d73..27093e93 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java @@ -1,6 +1,6 @@ package au.org.aodn.ogcapi.server.core.configuration; -import au.org.aodn.ogcapi.server.core.service.AWSBatchService; +import au.org.aodn.ogcapi.server.processes.RestServices; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -25,7 +25,7 @@ public BatchClient batchClient() { } @Bean - public AWSBatchService awsBatchService(BatchClient batchClient, ObjectMapper objectMapper) { - return new AWSBatchService(batchClient, objectMapper); + public RestServices awsBatchService(BatchClient batchClient, ObjectMapper objectMapper) { + return new RestServices(batchClient, objectMapper); } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java index 2291edaa..c56913dc 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java @@ -8,7 +8,6 @@ import au.org.aodn.ogcapi.processes.model.Results; import au.org.aodn.ogcapi.server.core.model.InlineValue; import au.org.aodn.ogcapi.server.core.model.enumeration.ProcessIdEnum; -import au.org.aodn.ogcapi.server.core.service.AWSBatchService; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; import io.swagger.v3.oas.annotations.media.Schema; @@ -25,7 +24,7 @@ public class RestApi implements ProcessesApi { @Autowired - private AWSBatchService awsBatchService; + private RestServices restServices; @Override // because the produces value in the interface declaration includes "/_" which may @@ -44,7 +43,7 @@ public ResponseEntity execute( if (processID.equals(ProcessIdEnum.DOWNLOAD_DATASET.getValue())) { try { - var response = awsBatchService.downloadData( + var response = restServices.downloadData( (String) body.getInputs().get("collectionId"), (String) body.getInputs().get("start_date"), (String) body.getInputs().get("end_date"), diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java similarity index 96% rename from server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java rename to server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java index 6ffe2119..57599f3e 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AWSBatchService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java @@ -1,4 +1,4 @@ -package au.org.aodn.ogcapi.server.core.service; +package au.org.aodn.ogcapi.server.processes; import au.org.aodn.ogcapi.server.core.model.enumeration.DatasetDownloadEnums; import com.fasterxml.jackson.databind.ObjectMapper; @@ -14,12 +14,12 @@ import java.util.Map; @Slf4j -public class AWSBatchService { +public class RestServices { private final BatchClient batchClient; private final ObjectMapper objectMapper; - public AWSBatchService(BatchClient batchClient, ObjectMapper objectMapper) { + public RestServices(BatchClient batchClient, ObjectMapper objectMapper) { this.batchClient = batchClient; this.objectMapper = objectMapper; } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java index 1fc8caad..1b7cd560 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestApiTest.java @@ -5,7 +5,6 @@ import au.org.aodn.ogcapi.processes.model.Results; import au.org.aodn.ogcapi.server.core.model.InlineValue; import au.org.aodn.ogcapi.server.core.model.enumeration.ProcessIdEnum; -import au.org.aodn.ogcapi.server.core.service.AWSBatchService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -25,7 +24,7 @@ public class RestApiTest { @Mock - private AWSBatchService awsBatchService; + private RestServices restServices; @InjectMocks private RestApi restApi; @@ -49,7 +48,7 @@ public void setUp() { @Test public void testExecuteDownloadDatasetSuccess() { - when(awsBatchService.downloadData(anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString())) + when(restServices.downloadData(anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString())) .thenReturn(ResponseEntity.ok("Job submitted with ID: test-job-id")); ResponseEntity response = restApi.execute(ProcessIdEnum.DOWNLOAD_DATASET.getValue(), executeRequest); @@ -63,7 +62,7 @@ public void testExecuteDownloadDatasetSuccess() { @Test public void testExecuteDownloadDatasetError() { - when(awsBatchService.downloadData(anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString())) + when(restServices.downloadData(anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString())) .thenThrow(new RuntimeException("Error while getting dataset")); ResponseEntity response = restApi.execute(ProcessIdEnum.DOWNLOAD_DATASET.getValue(), executeRequest); diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestServicesTest.java similarity index 84% rename from server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java rename to server/src/test/java/au/org/aodn/ogcapi/server/processes/RestServicesTest.java index 560522dd..e9dcac18 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/common/AWSBatchServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/processes/RestServicesTest.java @@ -1,6 +1,5 @@ -package au.org.aodn.ogcapi.server.common; +package au.org.aodn.ogcapi.server.processes; -import au.org.aodn.ogcapi.server.core.service.AWSBatchService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.InjectMocks; @@ -15,13 +14,13 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; -public class AWSBatchServiceTest { +public class RestServicesTest { @Mock private BatchClient batchClient; @InjectMocks - private AWSBatchService awsBatchService; + private RestServices restServices; @BeforeEach public void setUp() { @@ -36,7 +35,7 @@ public void testDownloadDataSuccess() { when(batchClient.submitJob(any(SubmitJobRequest.class))).thenReturn(submitJobResponse); // Act - ResponseEntity response = awsBatchService.downloadData( + ResponseEntity response = restServices.downloadData( "id", "2021-01-01", "2021-01-31", "10.0", "20.0", "30.0", "40.0", "recipient@example.com"); // Assert @@ -50,7 +49,7 @@ public void testDownloadDataFailure() { when(batchClient.submitJob(any(SubmitJobRequest.class))).thenThrow(new RuntimeException("AWS Batch error")); // Act - ResponseEntity response = awsBatchService.downloadData( + ResponseEntity response = restServices.downloadData( "id", "2021-01-01", "2021-01-31", "10.0", "20.0", "30.0", "40.0", "recipient@example.com"); // Assert