Features/6270 add download api

server/pom.xml

@@ -20,6 +20,16 @@
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>batch</artifactId>
+            <version>2.29.52</version>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>auth</artifactId>
+            <version>2.29.52</version>
+        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter</artifactId>
@@ -71,6 +81,11 @@
             <artifactId>ogcapi-features-java</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
+        <dependency>
+            <groupId>au.org.aodn.ogcapi</groupId>
+            <artifactId>ogcapi-processes-java</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
         <dependency>
             <groupId>co.elastic.clients</groupId>
             <artifactId>elasticsearch-java</artifactId>

server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/AwsConfig.java

@@ -0,0 +1,31 @@
+package au.org.aodn.ogcapi.server.core.configuration;
+
+import au.org.aodn.ogcapi.server.processes.RestServices;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.batch.BatchClient;
+
+@Configuration
+public class AwsConfig {
+
+    @Value("${aws.region}")
+    private String awsRegion;
+
+    @Bean
+    public BatchClient batchClient() {
+        return BatchClient
+                .builder()
+                .region(Region.of(awsRegion))
+                .credentialsProvider(EnvironmentVariableCredentialsProvider.create())
+                .build();
+    }
+
+    @Bean
+    public RestServices awsBatchService(BatchClient batchClient, ObjectMapper objectMapper) {
+        return new RestServices(batchClient, objectMapper);
+    }
+}

server/src/main/java/au/org/aodn/ogcapi/server/core/model/InlineValue.java

@@ -0,0 +1,5 @@
+package au.org.aodn.ogcapi.server.core.model;
+
+import au.org.aodn.ogcapi.processes.model.InlineOrRefData;
+
+public record InlineValue(String message) implements InlineOrRefData {}

server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java

@@ -0,0 +1,49 @@
+package au.org.aodn.ogcapi.server.core.model.enumeration;
+
+import lombok.Getter;
+
+/**
+ * the values are used for aws batch's environment variables
+ */
+public class DatasetDownloadEnums {
+
+    @Getter
+    public enum Condition {
+        UUID("UUID"),
+        START_DATE("START_DATE"),
+        END_DATE("END_DATE"),
+        MIN_LATITUDE("MIN_LAT"),
+        MAX_LATITUDE("MAX_LAT"),
+        MIN_LONGITUDE("MIN_LON"),
+        MAX_LONGITUDE("MAX_LON"),
+        RECIPIENT("RECIPIENT");
+
+        private final String value;
+
+        Condition(String value) {
+            this.value = value;
+        }
+    }
+
+    @Getter
+    public enum JobDefinition {
+        GENERATE_CSV_DATA_FILE("generate-csv-data-file");
+
+        private final String value;
+
+        JobDefinition(String value) {
+            this.value = value;
+        }
+    }
+
+    @Getter
+    public enum JobQueue {
+        GENERATING_CSV_DATA_FILE("generate-csv-data-file");
+
+        private final String value;
+
+        JobQueue(String value) {
+            this.value = value;
+        }
+    }
+}

server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/ProcessIdEnum.java

@@ -0,0 +1,15 @@
+package au.org.aodn.ogcapi.server.core.model.enumeration;
+
+import lombok.Getter;
+
+@Getter
+public enum ProcessIdEnum {
+    DOWNLOAD_DATASET("download"),
+    ;
+
+    private final String value;
+
+    ProcessIdEnum(String value) {
+        this.value = value;
+    }
+}

server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java

@@ -0,0 +1,90 @@
+package au.org.aodn.ogcapi.server.processes;
+
+
+import au.org.aodn.ogcapi.processes.api.ProcessesApi;
+import au.org.aodn.ogcapi.processes.model.Execute;
+import au.org.aodn.ogcapi.processes.model.InlineResponse200;
+import au.org.aodn.ogcapi.processes.model.ProcessList;
+import au.org.aodn.ogcapi.processes.model.Results;
+import au.org.aodn.ogcapi.server.core.model.InlineValue;
+import au.org.aodn.ogcapi.server.core.model.enumeration.ProcessIdEnum;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.enums.ParameterIn;
+import io.swagger.v3.oas.annotations.media.Schema;
+import jakarta.validation.Valid;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+
+@Slf4j
+@RestController("ProcessesRestApi")
+@RequestMapping(value = "/api/v1/ogc")
+public class RestApi implements ProcessesApi {
+
+    @Autowired
+    private RestServices restServices;
+
+    @Override
+    // because the produces value in the interface declaration includes "/_" which may
+    // cause exception thrown sometimes. So i re-declared the produces value here
+    @RequestMapping(value = "/processes/{processID}/execution",
+            produces = { "application/json", "text/html" },
+            consumes = { "application/json" },
+            method = RequestMethod.POST)
+    public ResponseEntity<InlineResponse200> execute(
+            @Parameter(in = ParameterIn.PATH, required=true, schema=@Schema())
+            @PathVariable("processID")
+            String processID,
+            @Parameter(in = ParameterIn.DEFAULT, description = "Mandatory execute request JSON", required=true, schema=@Schema())
+            @Valid
+            @RequestBody Execute body){
+
+        if (processID.equals(ProcessIdEnum.DOWNLOAD_DATASET.getValue())) {
+            try {
+                var response = restServices.downloadData(
+                        (String) body.getInputs().get("collectionId"),
+                        (String) body.getInputs().get("start_date"),
+                        (String) body.getInputs().get("end_date"),
+                        (String) body.getInputs().get("min_lat"),
+                        (String) body.getInputs().get("min_lon"),
+                        (String) body.getInputs().get("max_lat"),
+                        (String) body.getInputs().get("max_lon"),
+                        (String) body.getInputs().get("recipient")
+                );
+
+                var value = new InlineValue(response.getBody());
+                var results = new Results();
+                results.put("message", value);
+
+                return ResponseEntity.ok(results);
+
+            } catch (Exception e) {
+                log.error(e.getMessage());
+                var response = new Results();
+                var value = new InlineValue("Error while getting dataset");
+                response.put("error", value);
+
+                return ResponseEntity.badRequest().body(response);
+            }
+        }
+
+        var response = new Results();
+        var value = new InlineValue("Unknown process ID: " + processID);
+        response.put("error", value);
+
+        return ResponseEntity.badRequest().body(response);
+    }
+
+
+    @Override
+    public ResponseEntity<au.org.aodn.ogcapi.processes.model.Process> getProcessDescription(String processID) {
+        return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build();
+    }
+
+    @Override
+    public ResponseEntity<ProcessList> getProcesses() {
+        return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build();
+    }
+}

server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java

@@ -0,0 +1,121 @@
+package au.org.aodn.ogcapi.server.processes;
+
+import au.org.aodn.ogcapi.server.core.model.enumeration.DatasetDownloadEnums;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.ResponseEntity;
+import software.amazon.awssdk.services.batch.BatchClient;
+import software.amazon.awssdk.services.batch.model.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Slf4j
+public class RestServices {
+
+    private final BatchClient batchClient;
+    private final ObjectMapper objectMapper;
+
+    public RestServices(BatchClient batchClient, ObjectMapper objectMapper) {
+        this.batchClient = batchClient;
+        this.objectMapper = objectMapper;
+    }
+
+    public ResponseEntity<String> downloadData(
+            String id,
+            String startDate,
+            String endDate,
+            String minLat,
+            String minLon,
+            String maxLat,
+            String maxLon,
+            String recipient
+    ) {
+        try {
+
+            Map<String, String> parameters = new HashMap<>();
+            parameters.put(DatasetDownloadEnums.Condition.UUID.getValue(), id);
+            parameters.put(DatasetDownloadEnums.Condition.START_DATE.getValue(), startDate);
+            parameters.put(DatasetDownloadEnums.Condition.END_DATE.getValue(), endDate);
+            parameters.put(DatasetDownloadEnums.Condition.MIN_LATITUDE.getValue(), minLat);
+            parameters.put(DatasetDownloadEnums.Condition.MIN_LONGITUDE.getValue(), minLon);
+            parameters.put(DatasetDownloadEnums.Condition.MAX_LATITUDE.getValue(), maxLat);
+            parameters.put(DatasetDownloadEnums.Condition.MAX_LONGITUDE.getValue(), maxLon);
+            parameters.put(DatasetDownloadEnums.Condition.RECIPIENT.getValue(), recipient);
+
+
+            String jobId = submitJob(
+                    "generating-data-file-for-" + recipient.replaceAll("[^a-zA-Z0-9-_]", "-"),
+                    DatasetDownloadEnums.JobQueue.GENERATING_CSV_DATA_FILE.getValue(),
+                    DatasetDownloadEnums.JobDefinition.GENERATE_CSV_DATA_FILE.getValue(),
+                    parameters);
+            log.info("Job submitted with ID: " + jobId);
+            return ResponseEntity.ok("Job submitted with ID: " + jobId);
+        } catch (Exception e) {
+
+            log.error("Error while getting dataset");
+            log.error(e.getMessage());
+            return ResponseEntity.badRequest().body("Error while getting dataset");
+        }
+    }
+
+    private String submitJob(String jobName, String jobQueue, String jobDefinition, Map<String, String> parameters) {
+
+        List<KeyValuePair> environmentVariables = parameters.entrySet().stream()
+                .map(entry ->
+                        KeyValuePair
+                                .builder()
+                                .name(entry.getKey())
+                                .value(entry.getValue())
+                                .build()
+                )
+                .toList();
+
+        SubmitJobRequest submitJobRequest = SubmitJobRequest.builder()
+                .jobName(jobName)
+                .jobQueue(jobQueue)
+                .jobDefinition(jobDefinition)
+                .parameters(parameters)
+                .containerOverrides(co -> co.environment(environmentVariables))
+                .build();
+
+        SubmitJobResponse submitJobResponse = batchClient.submitJob(submitJobRequest);
+        return submitJobResponse.jobId();
+    }
+
+
+
+    // TODO: This feature doesn't work yet. Will be implemented in the future as this one is not urgent
+    public boolean isJobQueueValid(String jobQueueName) throws IOException {
+
+        var remoteJobQueueDetail = getRemoteJobQueueBy(jobQueueName);
+        var localJobQueueDetail = getLocalJobQueueDetailBy(jobQueueName);
+
+        return remoteJobQueueDetail.equals(localJobQueueDetail);
+    }
+
+    public JobQueueDetail getRemoteJobQueueBy(String name) {
+        var request = DescribeJobQueuesRequest
+                .builder()
+                .jobQueues(name)
+                .build();
+
+        var jobQueues = batchClient.describeJobQueues(request).jobQueues();
+
+        if (jobQueues != null && jobQueues.size() == 1) {
+            return jobQueues.get(0);
+        }
+        return null;
+    }
+
+    public JobQueueDetail getLocalJobQueueDetailBy(String jobQueueName) throws IOException {
+        var configJsonPath = "server/src/main/java/au/org/aodn/ogcapi/server/processes/config/" + jobQueueName + ".json";
+        var jsonFile = new File(configJsonPath);
+        var jsonStr = objectMapper.writeValueAsString(objectMapper.readValue(jsonFile, Object.class));
+        return objectMapper.readValue(jsonStr, JobQueueDetail.class);
+    }
+
+}

server/src/main/java/au/org/aodn/ogcapi/server/processes/config/generate-csv-data-file.json

@@ -0,0 +1,18 @@
+{
+  "jobQueueName": "generate-csv-data-file",
+  "jobQueueArn": "arn:aws:batch:ap-southeast-2:704910415367:job-queue/generate-csv-data-file",
+  "state": "ENABLED",
+  "status": "VALID",
+  "statusReason": "JobQueue Healthy",
+  "priority": 1,
+  "computeEnvironmentOrder": [
+    {
+      "order": 1,
+      "computeEnvironment": "arn:aws:batch:ap-southeast-2:704910415367:compute-environment/generate-csv-data-file"
+    }
+  ],
+  "serviceEnvironmentOrder": [],
+  "jobQueueType": "ECS_FARGATE",
+  "tags": {},
+  "jobStateTimeLimitActions": []
+}

server/src/main/resources/application.yaml

@@ -19,6 +19,9 @@ elasticsearch:
       path: search_suggestions
       fields: abstract_phrases, parameter_vocabs_sayt, platform_vocabs_sayt, organisation_vocabs_sayt
 
+aws:
+  region: ap-southeast-2
+
 api:
   host: http://localhost:${server.port}