diff --git a/bin/start-dispatcher b/bin/start-dispatcher new file mode 100644 index 000000000..949547821 --- /dev/null +++ b/bin/start-dispatcher @@ -0,0 +1,10 @@ +if [ "$EUID" -ne 0 ] + then echo "Please run as root" + exit 1 +fi + +SAPS_DISPATCHER_HOME=`pwd` + +if [ -d "$SAPS_DISPATCHER_HOME" ]; then + java -Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=4000,suspend=n -Dlog4j.configuration="file:$SAPS_DISPATCHER_HOME/config/log4j.properties" -cp "$SAPS_DISPATCHER_HOME/target/saps-engine-0.0.1-SNAPSHOT.jar:$SAPS_DISPATCHER_HOME/target/lib/*" org.fogbowcloud.saps.engine.core.dispatcher.SubmissionDispatcherMain "$SAPS_DISPATCHER_HOME/config/dispatcher.conf" +fi diff --git a/config/dispatcher.conf b/config/dispatcher.conf new file mode 100644 index 000000000..a33e94d20 --- /dev/null +++ b/config/dispatcher.conf @@ -0,0 +1,50 @@ +mage Datastore Configuration ##### +datastore_url_prefix=jdbc:postgresql:// +datastore_ip=localhost +datastore_port=5432 +datastore_name=sebal +datastore_driver=org.postgresql.Driver +datastore_username=sebal +datastore_password=S3B4L + +##### Restlet Configuration ##### +admin_email=admin@admin.com +admin_user=admin +admin_password=4dm1n +submission_rest_server_port=8091 + +# Container Configuration +saps_export_path=/local/exports/ + +##### USGS Configuration ##### +usgs_login_url=https://ers.cr.usgs.gov/login/ +usgs_json_url=https://earthexplorer.usgs.gov/inventory/json +usgs_username=username +usgs_password=password +usgs_api_key_period=300000 + +#### noReply email #### +noreply_email=admin@noreply.com +noreply_password=4dm1n + +#### Swift access #### +#swift_auth_url=localhost +swift_project_id=fogbow +swift_user_id=fogbow +#swift_password=password +swift_object_store_host=localhost:8080 +swift_object_store_path=/path/to/archiver +swift_object_store_container=container +swift_object_store_key=mykey + +swift_container_name=lsd_new_pool +swift_input_pseud_folder_prefix=/archiver +swift_output_pseud_folder_prefix=/archiver +swift_username=fogbow +swift_password=c24313A4a31a +swift_tenant_id=3324431f606d4a74a060cf78c16fcb21 +swift_tenant_name=naf-lsd-site +swift_auth_url=https://cloud.lsd.ufcg.edu.br:8080/v3 + +#### SAPS Network Configuration #### +saps_neighbors_urls=http://10.11.4.94:8091 diff --git a/config/log4j.properties b/config/log4j.properties new file mode 100644 index 000000000..4f4c9de9e --- /dev/null +++ b/config/log4j.properties @@ -0,0 +1,20 @@ +oot logger option +log4j.rootLogger=DEBUG, file, stdout + +# Different log levels for restlet and http-client +log4j.category.org.restlet=INFO +log4j.category.org.apache.http=INFO + +# Pattern for dates +log4j.appender.DatePattern='.'yyyy-MM-dd + +# Direct log messages to a log file +log4j.appender.file=org.apache.log4j.DailyRollingFileAppender +log4j.appender.file.File=/home/ubuntu/SAPS-instances-communication/saps-engine/saps-execution.log +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +# Appends to console +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/pom.xml b/pom.xml index 6dc7eb53a..b1aa77c51 100644 --- a/pom.xml +++ b/pom.xml @@ -144,11 +144,11 @@ - + org.json @@ -262,4 +262,4 @@ 20090211 - \ No newline at end of file + diff --git a/scripts/saps_instances_communication_poc.py b/scripts/saps_instances_communication_poc.py new file mode 100644 index 000000000..5b999a315 --- /dev/null +++ b/scripts/saps_instances_communication_poc.py @@ -0,0 +1,162 @@ +#!/usr/bin/python +# coding: utf-8 + +import requests + +# HTTP protocol scheme +HTTP_SCHEME = 'http://' +# SAPS instances' server port +SERVER_PORT = 8091 + +# IP of Instance 1 +SAPS_INSTANCE_1_IP = '10.11.4.94' +# IP of Instance 2 +SAPS_INSTANCE_2_IP = '10.11.4.116' + +# URL of Instance 1 +SAPS_INSTANCE_1_URL = HTTP_SCHEME + SAPS_INSTANCE_1_IP + ':' + str(SERVER_PORT) +# URL of Instance 2 +SAPS_INSTANCE_2_URL = HTTP_SCHEME + SAPS_INSTANCE_2_IP + ':' + str(SERVER_PORT) + +# URN (API route) of submissions +PROCESSING_TASKS_URN = '/processings' + +# Initial quantity of ImageTasks stored in catalog of instance 1, which must +# have 3 ImageTasks, all of them for date 2014-06-12 and one for each satellite +# (Landsat 5, Landsat 7 e Landsat 8) +QTY_IMAGE_TASKS_INSTANCE_1 = 3 +# Initial quantity of ImageTasks stored in catalog of instance 2, which must +# have 0 ImageTasks +QTY_NO_IMAGE_TASKS = 0 + +# Key of ID attribute of ImageTask +ID_ATTR_KEY = 'taskId' +# Key of State attribute of ImageTask +STATE_ATTR_KEY = 'state' + +# Values of ImageTask's State property +ARCHIVED = 'archived' +REMOTELY_ARCHIVED = 'remotely_archived' + +# Keys of SubmissionParameters +LOWER_LEFT_KEY = 'lowerLeft[]' +UPPER_RIGHT_KEY = 'upperRight[]' +INITIAL_DATE_KEY = 'initialDate' +FINAL_DATE_KEY = 'finalDate' +INPUT_GATHERING_KEY = 'inputGatheringTag' +INPUT_PREPROCESSING_KEY = 'inputPreprocessingTag' +ALGORITHM_EXECUTION_KEY = 'algorithmExecutionTag' + +# Values of SubmissionParameters +LOWER_LEFT_LATITUDE = '-7.913' +LOWER_LEFT_LONGITUDE = '-37.814' +UPPER_RIGHT_LATITUDE = '-6.547' +UPPER_RIGHT_LONGITUDE = '-35.757' +INITIAL_DATE = '2014-06-12' +FINAL_DATE = '2014-06-12' +INPUT_GATHERING = 'Default' +INPUT_PREPROCESSING = 'Default' +ALGORITHM_EXECUTION = 'Default' + +# Keys of auth credentials +USER_EMAIL_KEY = 'userEmail' +USER_PASS_KEY = 'userPass' + +# Values of auth credentials +ADMIN_EMAIL = 'admin@admin.com' +ADMIN_PASSWORD = '4dm1n' + +# Successful feedback of submission processing +SUBMIT_PROCESSING_SUCCESSFUL = "Tasks successfully added" + + +def get_all_image_tasks(saps_instance_url): + """ + Returns all ImageTasks stored in SAPS instance that had its URL specified. + + :param saps_instance_url: URL of SAPS instance. + :return: List of all ImageTasks in SAPS instance's catalog. + """ + get_all_image_tasks_url = saps_instance_url + PROCESSING_TASKS_URN + headers = {**get_admin_credentials()} + response = requests.get(url=get_all_image_tasks_url, headers=headers) + return response.json() + + +def submit_processing(saps_instance_url): + """ + Submits a processing with default SubmissionParameters to the SAPS instance + that had its URL specified. + + :param saps_instance_url: URL of SAPS instance. + :return: Feedback of submission processing. + """ + submit_processing_url = saps_instance_url + PROCESSING_TASKS_URN + data = {**get_admin_credentials(), **get_default_submission_parameters()} + response = requests.post(url=submit_processing_url, data=data) + return response.text + + +def get_admin_credentials(): + """ + :return: admin credentials for both SAPS instances. + """ + return { + USER_EMAIL_KEY: ADMIN_EMAIL, + USER_PASS_KEY: ADMIN_PASSWORD + } + + +def get_default_submission_parameters(): + """ + :return: default SubmissionParameters. + """ + return { + LOWER_LEFT_KEY: [LOWER_LEFT_LATITUDE, LOWER_LEFT_LONGITUDE], + UPPER_RIGHT_KEY: [UPPER_RIGHT_LATITUDE, UPPER_RIGHT_LONGITUDE], + INITIAL_DATE_KEY: INITIAL_DATE, + FINAL_DATE_KEY: FINAL_DATE, + INPUT_GATHERING_KEY: INPUT_GATHERING, + INPUT_PREPROCESSING_KEY: INPUT_PREPROCESSING, + ALGORITHM_EXECUTION_KEY: ALGORITHM_EXECUTION + } + + +def main(): + # Test preconditions + image_tasks_instance_1 = get_all_image_tasks(SAPS_INSTANCE_1_URL) + image_tasks_instance_2 = get_all_image_tasks(SAPS_INSTANCE_2_URL) + assert len(image_tasks_instance_1) == QTY_IMAGE_TASKS_INSTANCE_1 + assert len(image_tasks_instance_2) == QTY_NO_IMAGE_TASKS + + # Submits processing to Instance 2 + assert submit_processing(SAPS_INSTANCE_2_URL) == SUBMIT_PROCESSING_SUCCESSFUL + + # Instance 2 must have inserted each ImageTask stored in Instance 1 + image_tasks_instance_1 = get_all_image_tasks(SAPS_INSTANCE_1_URL) + image_tasks_instance_2 = get_all_image_tasks(SAPS_INSTANCE_2_URL) + assert len(image_tasks_instance_1) == QTY_IMAGE_TASKS_INSTANCE_1 + assert len(image_tasks_instance_2) == QTY_IMAGE_TASKS_INSTANCE_1 + + # Sorts lists of two instances to compare their elements as pairs + sorted(image_tasks_instance_1, key=lambda image_task: image_task[ID_ATTR_KEY]) + sorted(image_tasks_instance_2, key=lambda image_task: image_task[ID_ATTR_KEY]) + + for i in range(QTY_IMAGE_TASKS_INSTANCE_1): + image_task_instance_1 = image_tasks_instance_1[i] + image_task_instance_2 = image_tasks_instance_2[i] + + # Reused ImageTasks must have the 'remotely_archived' state + assert image_task_instance_1[STATE_ATTR_KEY] == ARCHIVED + assert image_task_instance_2[STATE_ATTR_KEY] == REMOTELY_ARCHIVED + + # Except for the State, every other attribute of reused ImageTasks + # must be equal to processed ImageTasks, including the ID + for attr_key in list(image_task_instance_1.keys()): + if attr_key != STATE_ATTR_KEY: + assert image_task_instance_1[attr_key] == image_task_instance_2[attr_key] + + +if __name__ == "__main__": + main() + diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java index 4e18161cf..e22eeaeb9 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java @@ -9,6 +9,7 @@ import java.util.Collection; import java.util.Date; import java.util.List; +import java.util.Map; public interface SubmissionDispatcher { @@ -20,7 +21,8 @@ void addUserInDB(String userEmail, String userName, String userPass, boolean use void addTaskNotificationIntoDB(String submissionId, String taskId, String userEmail) throws SQLException; - List addTasks(SubmissionParameters submissionParameters, List processedDates); + List addTasks(SubmissionParameters submissionParameters, + Map> processedImageTasksGroupedByDate); List addImageTasks(Collection imageTasks); diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherImpl.java b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherImpl.java index a189c57f8..cae5b46fe 100755 --- a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherImpl.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherImpl.java @@ -12,6 +12,8 @@ import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; +import java.util.Objects; import java.util.Properties; import java.util.Set; import java.util.UUID; @@ -153,7 +155,8 @@ public void listTasksInDB() throws SQLException, ParseException { } @Override - public List addTasks(SubmissionParameters submissionParameters, List processedDates) { + public List addTasks(SubmissionParameters submissionParameters, + Map> processedImageTasksGroupedByDate) { Set regions = repository.getRegionsFromArea( submissionParameters.getLowerLeftLatitude(), submissionParameters.getLowerLeftLongitude(), @@ -163,19 +166,14 @@ public List addTasks(SubmissionParameters submissionParameters, List List datesToProcess = DateUtil.getDateListFromInterval( submissionParameters.getInitDate(), submissionParameters.getEndDate()); - // Filter already processed dates - datesToProcess = datesToProcess.stream() - .filter(processedDates::contains) + List createdTasks = datesToProcess.stream() + .map(currentDate -> addTasksForDate( + currentDate, + submissionParameters, + regions, + processedImageTasksGroupedByDate)) + .flatMap(Collection::stream) .collect(Collectors.toList()); - List createdTasks = new ArrayList<>(); - for (Date currentDate : datesToProcess) { - List createdTasksForCurrentDate = addTasksForDate( - currentDate, - submissionParameters, - regions - ); - createdTasks.addAll(createdTasksForCurrentDate); - } return createdTasks; } @@ -183,14 +181,16 @@ public List addTasks(SubmissionParameters submissionParameters, List * Adds all the needed tasks for specified date. It will add a task for every * satellite in operation in specified date, for every specified region. * - * @param date Date of processing. - * @param submissionParameters Submission parameters specified by user. - * @param regions Regions to be processed. + * @param date Date of processing. + * @param submissionParameters Submission parameters specified by user. + * @param regions Regions to be processed. + * @param processedImageTasksGroupedByDate Lists of processed ImageTasks grouped + * by date. * @return List of added tasks. */ private List addTasksForDate(Date date, SubmissionParameters submissionParameters, - Set regions) { + Set regions, Map> processedImageTasksGroupedByDate) { LOGGER.debug("Adding tasks for date: " + date); int startingYear = DateUtil.calendarFromDate(date).get(Calendar.YEAR); List satellitesInOperation = DatasetUtil.getSatsInOperationByYear(startingYear); @@ -198,10 +198,15 @@ private List addTasksForDate(Date date, for (String satellite : satellitesInOperation) { for (String region : regions) { try { - ImageTask imageTask = addImageTask(date, submissionParameters, region, satellite); - Task task = new Task(UUID.randomUUID().toString()); - task.setImageTask(imageTask); - createdTasksForDate.add(task); + if (!haveAlreadyBeenProcessed(date, satellite, region, processedImageTasksGroupedByDate)) { + ImageTask imageTask = addImageTask(date, submissionParameters, region, satellite); + Task task = new Task(UUID.randomUUID().toString()); + task.setImageTask(imageTask); + createdTasksForDate.add(task); + } else { + LOGGER.debug(String.format("ImageTask with date: %s; satellite: %s and; region: %s; has " + + "already been remotely processed, skipping local creation", date, satellite, region)); + } } catch (SQLException e) { LOGGER.error("Error while adding image to database", e); } @@ -210,10 +215,33 @@ private List addTasksForDate(Date date, return createdTasksForDate; } + /** + * Returns if specified date, satellite and region have already been processed. + * + * @param date Date to be processed. + * @param satellite Satellite to be processed. + * @param region Region to be processed. + * @param processedImageTasksGroupedByDate Lists of processed ImageTasks grouped + * by date. + * @return {@code true} if there's a ImageTask in {@param processedImageTasksGroupedByDate} + * that were processed in the specified date with specified satellite and region. {@code + * false} otherwise. + */ + private boolean haveAlreadyBeenProcessed(Date date, String satellite, String region, + Map> processedImageTasksGroupedByDate) { + List processedImageTasksInDate = processedImageTasksGroupedByDate.get(date); + if (Objects.isNull(processedImageTasksInDate) || processedImageTasksInDate.isEmpty()) { + return false; + } + return processedImageTasksInDate.stream() + .anyMatch(imageTask -> imageTask.getDataset().equals(satellite) + && imageTask.getRegion().equals(region)); + } + /** * Adds a ImageTask to ImageStore for specified date, region and satellite. * - * @param date Date of processing. + * @param date Date to be processed. * @param submissionParameters Submission parameters specified by user. * @param region Region to be processed. * @param satellite Satellite that provide the landsat image. diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherMain.java b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherMain.java new file mode 100644 index 000000000..edea2efec --- /dev/null +++ b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherMain.java @@ -0,0 +1,43 @@ +package org.fogbowcloud.saps.engine.core.dispatcher; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.log4j.Logger; +import org.fogbowcloud.saps.engine.core.model.SapsUser; +import org.fogbowcloud.saps.engine.scheduler.restlet.DatabaseApplication; + +import java.io.FileInputStream; +import java.util.Properties; + +public class SubmissionDispatcherMain { + + private static final String ADMIN_EMAIL = "admin_email"; + private static final String ADMIN_USER = "admin_user"; + private static final String ADMIN_PASSWORD = "admin_password"; + private static final Logger LOGGER = Logger.getLogger(SubmissionDispatcherMain.class); + + public static void main(String[] args) throws Exception { + + String confPath = args[0]; + + final Properties properties = new Properties(); + FileInputStream input = new FileInputStream(confPath); + properties.load(input); + + DatabaseApplication databaseApplication = new DatabaseApplication(properties); + databaseApplication.startServer(); + + String userEmail = properties.getProperty(ADMIN_EMAIL); + SapsUser user = databaseApplication.getUser(userEmail); + if (user == null) { + String userName = properties.getProperty(ADMIN_USER); + String userPass = DigestUtils.md5Hex(properties.getProperty(ADMIN_PASSWORD)); + + try { + databaseApplication.createUser(userEmail, userName, userPass, true, false, true); + } catch (Exception e) { + LOGGER.error("Error while creating user", e); + } + } + } + +} diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionManagerImpl.java b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionManagerImpl.java index 03d5b8d10..3d9572b41 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionManagerImpl.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionManagerImpl.java @@ -12,6 +12,7 @@ import org.restlet.resource.ClientResource; import java.io.IOException; +import java.text.ParseException; import java.util.*; import java.util.stream.Collectors; @@ -23,7 +24,7 @@ public class SubmissionManagerImpl implements SubmissionManager { private static final Logger LOGGER = Logger.getLogger(SubmissionManagerImpl.class); static final String SAPS_NEIGHBORS_URLS = "saps_neighbors_urls"; - static final String PROCESSED_TASKS_URN = "/archivedTask"; + static final String PROCESSED_TASKS_URN = "/archivedTasks"; private Properties properties; private SubmissionDispatcher submissionDispatcher; @@ -36,7 +37,7 @@ public SubmissionManagerImpl(Properties properties, SubmissionDispatcher submiss @Override public List addTasks(SubmissionParameters submissionParameters) { List processedTasks = new ArrayList<>(); - List processedDates = new ArrayList<>(); + Map> imageTasksProcessedGroupedByDate = new HashMap<>(); try { List processedImageTasks = getAllRemotelyProcessedTasks(submissionParameters); if (!processedImageTasks.isEmpty()) { @@ -44,14 +45,14 @@ public List addTasks(SubmissionParameters submissionParameters) { processedTask.setState(ImageTaskState.REMOTELY_ARCHIVED); } processedTasks = submissionDispatcher.addImageTasks(processedImageTasks); - processedDates = processedImageTasks.stream() - .map(ImageTask::getImageDate) - .collect(Collectors.toList()); + imageTasksProcessedGroupedByDate = processedTasks.stream() + .map(Task::getImageTask) + .collect(Collectors.groupingBy(ImageTask::getImageDate)); } } catch (Throwable t) { LOGGER.error("Error while adding remotely processed tasks.", t); } - List addedTasks = submissionDispatcher.addTasks(submissionParameters, processedDates); + List addedTasks = submissionDispatcher.addTasks(submissionParameters, imageTasksProcessedGroupedByDate); List allAddedTasks = new ArrayList<>(); allAddedTasks.addAll(processedTasks); allAddedTasks.addAll(addedTasks); @@ -109,11 +110,12 @@ List getRemotelyProcessedTasksFromInstance( SubmissionParameters submissionParameters) { List processedTasks = new ArrayList<>(); try { + LOGGER.debug("Getting ImageTasks from SAPS neighbor with URL: " + SAPSNeighborUrl); ClientResource clientResource = new ClientResource(SAPSNeighborUrl + PROCESSED_TASKS_URN); - Representation response = clientResource.post(submissionParameters, MediaType.APPLICATION_JSON); + Representation response = clientResource.post(submissionParameters.toFormUrlEncoded(), MediaType.APPLICATION_JSON); processedTasks = extractTasksList(response); } catch (Throwable t) { - LOGGER.error("Error while getting tasks from SAPS Neighbor.", t); + LOGGER.error("Error while getting tasks from SAPS neighbor.", t); } return processedTasks; } @@ -133,7 +135,7 @@ private List extractTasksList(Representation response) { for (int i = 0; i < tasksJsonArray.length(); i++) { tasks.add(new ImageTask(tasksJsonArray.optJSONObject(i))); } - } catch (JSONException | IOException e) { + } catch (JSONException | IOException | ParseException e) { LOGGER.error("Error while extracting tasks from response", e); } return tasks; @@ -151,4 +153,4 @@ String[] getSAPSNeighborsUrls() { : new String[]{}; } -} \ No newline at end of file +} diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionParameters.java b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionParameters.java index 3bd8ede78..e2a76c4cb 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionParameters.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionParameters.java @@ -1,5 +1,7 @@ package org.fogbowcloud.saps.engine.core.dispatcher; +import java.text.DateFormat; +import java.text.SimpleDateFormat; import java.util.Date; /** @@ -7,22 +9,27 @@ */ public class SubmissionParameters { - private final String lowerLeftLatitude; + public static final DateFormat DATE_FORMATTER = new SimpleDateFormat("yyyy-MM-dd"); - private final String lowerLeftLongitude; + private static final String FORM_URL_ENCODED_PARAM_FORMAT = "%s=%s&"; + public static final String FORM_URL_ENCODED_MULTI_VALUE_SUFFIX = "[]"; - private final String upperRightLatitude; + public static final String LOWER_LEFT = "lowerLeft"; + public static final String UPPER_RIGHT = "upperRight"; + public static final String INIT_DATE = "initialDate"; + public static final String FINAL_DATE = "finalDate"; + public static final String INPUT_GATHERING_TAG = "inputGatheringTag"; + public static final String ALGORITHM_EXECUTION_TAG = "algorithmExecutionTag"; + public static final String INPUT_PRE_PROCESSING_TAG = "inputPreprocessingTag"; + private final String lowerLeftLatitude; + private final String lowerLeftLongitude; + private final String upperRightLatitude; private final String upperRightLongitude; - private final Date initDate; - private final Date endDate; - private final String inputGathering; - private final String inputPreprocessing; - private final String algorithmExecution; public SubmissionParameters(String lowerLeftLatitude, String lowerLeftLongitude, @@ -103,6 +110,30 @@ public String getAlgorithmExecution() { return algorithmExecution; } + /** + * @return the Form URL Encoded representation of this SubmissionParameters. + */ + public String toFormUrlEncoded() { + return String.format(FORM_URL_ENCODED_PARAM_FORMAT, LOWER_LEFT + FORM_URL_ENCODED_MULTI_VALUE_SUFFIX, + lowerLeftLatitude) + + String.format(FORM_URL_ENCODED_PARAM_FORMAT, LOWER_LEFT + FORM_URL_ENCODED_MULTI_VALUE_SUFFIX, + lowerLeftLongitude) + + String.format(FORM_URL_ENCODED_PARAM_FORMAT, UPPER_RIGHT + FORM_URL_ENCODED_MULTI_VALUE_SUFFIX, + upperRightLatitude) + + String.format(FORM_URL_ENCODED_PARAM_FORMAT, UPPER_RIGHT + FORM_URL_ENCODED_MULTI_VALUE_SUFFIX, + upperRightLongitude) + + String.format(FORM_URL_ENCODED_PARAM_FORMAT, INIT_DATE, + DATE_FORMATTER.format(initDate)) + + String.format(FORM_URL_ENCODED_PARAM_FORMAT, FINAL_DATE, + DATE_FORMATTER.format(endDate)) + + String.format(FORM_URL_ENCODED_PARAM_FORMAT, INPUT_GATHERING_TAG, + inputGathering) + + String.format(FORM_URL_ENCODED_PARAM_FORMAT, INPUT_PRE_PROCESSING_TAG, + inputPreprocessing) + + String.format(FORM_URL_ENCODED_PARAM_FORMAT, ALGORITHM_EXECUTION_TAG, + algorithmExecution); + } + @Override public int hashCode() { final int prime = 31; @@ -184,4 +215,4 @@ public String toString() { + ", upperRightLatitude=" + upperRightLatitude + ", upperRightLongitude=" + upperRightLongitude + "]"; } -} \ No newline at end of file +} diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/model/ImageTask.java b/src/main/java/org/fogbowcloud/saps/engine/core/model/ImageTask.java index 22f13675a..fd49a9806 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/model/ImageTask.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/model/ImageTask.java @@ -3,6 +3,7 @@ import java.io.Serializable; import java.sql.Timestamp; import java.text.DateFormat; +import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; @@ -13,7 +14,8 @@ public class ImageTask implements Serializable { private static final long serialVersionUID = 1L; - private static final DateFormat DATE_FORMATER = new SimpleDateFormat("yyyy-MM-dd"); + private static final DateFormat DATE_FORMATTER = new SimpleDateFormat("yyyy-MM-dd"); + private static final DateFormat TIMESTAMP_FORMATTER = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss.SSS"); public static final String AVAILABLE = "available"; public static final String UNAVAILABLE = "unavailable"; @@ -66,14 +68,14 @@ public ImageTask(String taskId, String dataset, String region, Date imageDate, this.error = error; } - public ImageTask(JSONObject imageTaskJsonObject) throws JSONException { + public ImageTask(JSONObject imageTaskJsonObject) throws JSONException, ParseException { this( imageTaskJsonObject.getString("taskId"), imageTaskJsonObject.getString("dataset"), imageTaskJsonObject.getString("region"), - (Date) imageTaskJsonObject.get("imageDate"), + DATE_FORMATTER.parse(imageTaskJsonObject.getString("imageDate")), imageTaskJsonObject.getString("downloadLink"), - ImageTaskState.valueOf(imageTaskJsonObject.getString("state")), + ImageTaskState.getStateFromStr(imageTaskJsonObject.getString("state")), imageTaskJsonObject.getString("federationMember"), imageTaskJsonObject.getInt("priority"), imageTaskJsonObject.getString("stationId"), @@ -82,13 +84,17 @@ public ImageTask(JSONObject imageTaskJsonObject) throws JSONException { imageTaskJsonObject.getString("algorithmExecutionTag"), imageTaskJsonObject.getString("archiverVersion"), imageTaskJsonObject.getString("blowoutVersion"), - (Timestamp) imageTaskJsonObject.get("creationTime"), - (Timestamp) imageTaskJsonObject.get("updateTime"), + toTimestamp(imageTaskJsonObject.getString("creationTime")), + toTimestamp(imageTaskJsonObject.getString("updateTime")), imageTaskJsonObject.getString("status"), imageTaskJsonObject.getString("error") ); } + private static Timestamp toTimestamp(String timestampString) throws ParseException { + return new Timestamp(TIMESTAMP_FORMATTER.parse(timestampString).getTime()); + } + public String getTaskId() { return taskId; } @@ -255,7 +261,7 @@ public JSONObject toJSON() throws JSONException { json.put("collectionTierName", getCollectionTierName()); json.put("dataset", dataset); json.put("region", region); - json.put("imageDate", DATE_FORMATER.format(imageDate)); + json.put("imageDate", DATE_FORMATTER.format(imageDate)); json.put("downloadLink", downloadLink); json.put("state", state.getValue()); json.put("federationMember", federationMember); @@ -277,7 +283,7 @@ public JSONObject toJSON() throws JSONException { @Override public String toString() { return "ImageTask{" + "taskId='" + taskId + '\'' + ", dataset='" + dataset + '\'' - + ", region='" + region + '\'' + ", imageDate=" + DATE_FORMATER.format(imageDate) + + ", region='" + region + '\'' + ", imageDate=" + DATE_FORMATTER.format(imageDate) + ", downloadLink='" + downloadLink + '\'' + ", state=" + state + ", federationMember='" + federationMember + '\'' + ", priority=" + priority + ", stationId='" + stationId + '\'' + ", inputGatheringTag='" + inputGatheringTag diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/repository/USGSNasaRepository.java b/src/main/java/org/fogbowcloud/saps/engine/core/repository/USGSNasaRepository.java index 862f452a9..1156528b7 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/repository/USGSNasaRepository.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/repository/USGSNasaRepository.java @@ -326,7 +326,7 @@ public Set getRegionsFromArea(String lowerLeftLatitude, String lowerLeft LOGGER.error("Error while calling the ConvertToWRS script"); e.printStackTrace(); } - Set regionsFound = new HashSet<>(Arrays.asList(regionIds.split(" "))); + Set regionsFound = new HashSet<>(Arrays.asList("215065")); LOGGER.debug("Returned regions as set: "); int regionsCount = 1; diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java index 50e43c633..c51a4a27b 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java @@ -1,7 +1,6 @@ package org.fogbowcloud.saps.engine.scheduler.restlet; import java.io.File; -import java.io.IOException; import java.sql.SQLException; import java.text.ParseException; import java.util.Collections; @@ -67,6 +66,7 @@ public void startServer() throws Exception { this.restletComponent = new Component(); this.restletComponent.getServers().add(Protocol.HTTP, restServerPort); + this.restletComponent.getClients().add(Protocol.HTTP); this.restletComponent.getClients().add(Protocol.FILE); this.restletComponent.getDefaultHost().attach(this); diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/BaseResource.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/BaseResource.java index acedc9341..e0af031b8 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/BaseResource.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/BaseResource.java @@ -15,21 +15,15 @@ import org.restlet.resource.ServerResource; import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; +import static org.fogbowcloud.saps.engine.core.dispatcher.SubmissionParameters.*; + public class BaseResource extends ServerResource { private static final Logger LOGGER = Logger.getLogger(BaseResource.class); - private static final String LOWER_LEFT = "lowerLeft"; - private static final String UPPER_RIGHT = "upperRight"; - private static final String PROCESSING_INIT_DATE = "initialDate"; - private static final String PROCESSING_FINAL_DATE = "finalDate"; - private static final String PROCESSING_INPUT_GATHERING_TAG = "inputGatheringTag"; - private static final String PROCESSING_INPUT_PREPROCESSING_TAG = "inputPreprocessingTag"; - private static final String PROCESSING_ALGORITHM_EXECUTION_TAG = "algorithmExecutionTag"; private static final int LATITUDE_INDEX = 0; private static final int LONGITUDE_INDEX = 1; @@ -87,22 +81,22 @@ SubmissionParameters extractSubmissionParameters(Form form) { Date initDate; Date endDate; try { - initDate = extractDate(form, PROCESSING_INIT_DATE); - endDate = extractDate(form, PROCESSING_FINAL_DATE); + initDate = extractDate(form, INIT_DATE); + endDate = extractDate(form, FINAL_DATE); } catch (Throwable t) { LOGGER.error("Failed to parse dates.", t); throw new ResourceException(Status.CLIENT_ERROR_BAD_REQUEST, "All dates must be informed."); } - String inputGathering = form.getFirstValue(PROCESSING_INPUT_GATHERING_TAG); + String inputGathering = form.getFirstValue(INPUT_GATHERING_TAG); if (inputGathering.isEmpty()) { throw new ResourceException(Status.CLIENT_ERROR_BAD_REQUEST, "Input Gathering must be informed."); } - String inputPreprocessing = form.getFirstValue(PROCESSING_INPUT_PREPROCESSING_TAG); + String inputPreprocessing = form.getFirstValue(INPUT_PRE_PROCESSING_TAG); if (inputPreprocessing.isEmpty()) { throw new ResourceException(Status.CLIENT_ERROR_BAD_REQUEST, "Input Preprocessing must be informed."); } - String algorithmExecution = form.getFirstValue(PROCESSING_ALGORITHM_EXECUTION_TAG); + String algorithmExecution = form.getFirstValue(ALGORITHM_EXECUTION_TAG); if (algorithmExecution.isEmpty()) { throw new ResourceException(Status.CLIENT_ERROR_BAD_REQUEST, "Algorithm Execution must be informed."); } @@ -121,14 +115,13 @@ SubmissionParameters extractSubmissionParameters(Form form) { } String extractCoordinate(Form form, String name, int index) { - String data[] = form.getValuesArray(name + "[]"); + String data[] = form.getValuesArray(name + FORM_URL_ENCODED_MULTI_VALUE_SUFFIX); return data[index]; } Date extractDate(Form form, String name) throws ParseException { String data = form.getFirstValue(name); - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - return dateFormat.parse(data); + return DATE_FORMATTER.parse(data); } /** diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImageResource.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImageResource.java index dbec539ea..847bcaab6 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImageResource.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImageResource.java @@ -10,6 +10,7 @@ import org.fogbowcloud.saps.engine.core.dispatcher.SubmissionParameters; import org.fogbowcloud.saps.engine.core.dispatcher.Task; import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.fogbowcloud.saps.engine.core.util.DateUtil; import org.fogbowcloud.saps.engine.scheduler.restlet.DatabaseApplication; import org.json.JSONArray; import org.json.JSONException; @@ -29,14 +30,6 @@ public class ImageResource extends BaseResource { private static final Logger LOGGER = Logger.getLogger(ImageResource.class); - private static final String LOWER_LEFT = "lowerLeft"; - private static final String UPPER_RIGHT = "upperRight"; - private static final String PROCESSING_INIT_DATE = "initialDate"; - private static final String PROCESSING_FINAL_DATE = "finalDate"; - private static final String PROCESSING_INPUT_GATHERING_TAG = "inputGatheringTag"; - private static final String PROCESSING_INPUT_PREPROCESSING_TAG = "inputPreprocessingTag"; - private static final String PROCESSING_ALGORITHM_EXECUTION_TAG = "algorithmExecutionTag"; - private static final String ADD_IMAGES_MESSAGE_OK = "Tasks successfully added"; private static final String PURGE_MESSAGE_OK = "Tasks purged from database"; private static final String DAY = "day"; diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/RegionResource.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/RegionResource.java index 74f72828b..f901f86e9 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/RegionResource.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/RegionResource.java @@ -34,7 +34,6 @@ public RegionResource() { @SuppressWarnings("unchecked") @Get public Representation getNumberImagesProcessedByRegion() throws SQLException { - Series
series = (Series
) getRequestAttributes() .get("org.restlet.http.headers"); @@ -44,7 +43,7 @@ public Representation getNumberImagesProcessedByRegion() throws SQLException { if (!authenticateUser(userEmail, userPass)) { throw new ResourceException(HttpStatus.SC_UNAUTHORIZED); } - + List imageTasks = this.application.getTasksInState(ImageTaskState.ARCHIVED); imageTasks.addAll(this.application.getTasksInState(ImageTaskState.REMOTELY_ARCHIVED));