Proof of Concept of communication between SAPS instances

bin/start-dispatcher

@@ -0,0 +1,10 @@
+if [ "$EUID" -ne 0 ]
+  then echo "Please run as root"
+  exit 1
+fi
+
+SAPS_DISPATCHER_HOME=`pwd`
+
+if [ -d "$SAPS_DISPATCHER_HOME" ]; then
+  java -Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=4000,suspend=n -Dlog4j.configuration="file:$SAPS_DISPATCHER_HOME/config/log4j.properties" -cp "$SAPS_DISPATCHER_HOME/target/saps-engine-0.0.1-SNAPSHOT.jar:$SAPS_DISPATCHER_HOME/target/lib/*" org.fogbowcloud.saps.engine.core.dispatcher.SubmissionDispatcherMain "$SAPS_DISPATCHER_HOME/config/dispatcher.conf"
+fi

config/dispatcher.conf

@@ -0,0 +1,50 @@
+mage Datastore Configuration #####
+datastore_url_prefix=jdbc:postgresql://
+datastore_ip=localhost
+datastore_port=5432
+datastore_name=sebal
+datastore_driver=org.postgresql.Driver
+datastore_username=sebal
+datastore_password=S3B4L
+
+##### Restlet Configuration #####
+admin_email=admin@admin.com
+admin_user=admin
+admin_password=4dm1n
+submission_rest_server_port=8091
+
+# Container Configuration
+saps_export_path=/local/exports/
+
+##### USGS Configuration #####
+usgs_login_url=https://ers.cr.usgs.gov/login/
+usgs_json_url=https://earthexplorer.usgs.gov/inventory/json
+usgs_username=username
+usgs_password=password
+usgs_api_key_period=300000
+
+#### noReply email ####
+noreply_email=admin@noreply.com
+noreply_password=4dm1n
+
+#### Swift access ####
+#swift_auth_url=localhost
+swift_project_id=fogbow
+swift_user_id=fogbow
+#swift_password=password
+swift_object_store_host=localhost:8080
+swift_object_store_path=/path/to/archiver
+swift_object_store_container=container
+swift_object_store_key=mykey
+
+swift_container_name=lsd_new_pool
+swift_input_pseud_folder_prefix=/archiver
+swift_output_pseud_folder_prefix=/archiver
+swift_username=fogbow
+swift_password=c24313A4a31a
+swift_tenant_id=3324431f606d4a74a060cf78c16fcb21
+swift_tenant_name=naf-lsd-site
+swift_auth_url=https://cloud.lsd.ufcg.edu.br:8080/v3
+
+#### SAPS Network Configuration ####
+saps_neighbors_urls=http://10.11.4.94:8091

config/log4j.properties

@@ -0,0 +1,20 @@
+oot logger option
+log4j.rootLogger=DEBUG, file, stdout
+
+# Different log levels for restlet and http-client
+log4j.category.org.restlet=INFO
+log4j.category.org.apache.http=INFO
+
+# Pattern for dates
+log4j.appender.DatePattern='.'yyyy-MM-dd
+
+# Direct log messages to a log file
+log4j.appender.file=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.file.File=/home/ubuntu/SAPS-instances-communication/saps-engine/saps-execution.log
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+# Appends to console
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n

pom.xml

@@ -144,11 +144,11 @@
 			</exclusions>
 		</dependency>
 
-		<dependency>
+		<!--<dependency>
 			<groupId>blowout</groupId>
 			<artifactId>blowout</artifactId>
 			<version>0.0.1-SNAPSHOT</version>
-		</dependency>
+		</dependency>-->
 
 		<dependency>
 			<groupId>org.json</groupId>
@@ -262,4 +262,4 @@
     		<version>20090211</version>
 	</dependency>
 	</dependencies>
-</project>
+</project>

scripts/saps_instances_communication_poc.py

@@ -0,0 +1,162 @@
+#!/usr/bin/python
+# coding: utf-8
+
+import requests
+
+# HTTP protocol scheme
+HTTP_SCHEME = 'http://'
+# SAPS instances' server port
+SERVER_PORT = 8091
+
+# IP of Instance 1
+SAPS_INSTANCE_1_IP = '10.11.4.94'
+# IP of Instance 2
+SAPS_INSTANCE_2_IP = '10.11.4.116'
+
+# URL of Instance 1
+SAPS_INSTANCE_1_URL = HTTP_SCHEME + SAPS_INSTANCE_1_IP + ':' + str(SERVER_PORT)
+# URL of Instance 2
+SAPS_INSTANCE_2_URL = HTTP_SCHEME + SAPS_INSTANCE_2_IP + ':' + str(SERVER_PORT)
+
+# URN (API route) of submissions
+PROCESSING_TASKS_URN = '/processings'
+
+# Initial quantity of ImageTasks stored in catalog of instance 1, which must
+# have 3 ImageTasks, all of them for date 2014-06-12 and one for each satellite
+# (Landsat 5, Landsat 7 e Landsat 8)
+QTY_IMAGE_TASKS_INSTANCE_1 = 3
+# Initial quantity of ImageTasks stored in catalog of instance 2, which must
+# have 0 ImageTasks
+QTY_NO_IMAGE_TASKS = 0
+
+# Key of ID attribute of ImageTask
+ID_ATTR_KEY = 'taskId'
+# Key of State attribute of ImageTask
+STATE_ATTR_KEY = 'state'
+
+# Values of ImageTask's State property
+ARCHIVED = 'archived'
+REMOTELY_ARCHIVED = 'remotely_archived'
+
+# Keys of SubmissionParameters
+LOWER_LEFT_KEY = 'lowerLeft[]'
+UPPER_RIGHT_KEY = 'upperRight[]'
+INITIAL_DATE_KEY = 'initialDate'
+FINAL_DATE_KEY = 'finalDate'
+INPUT_GATHERING_KEY = 'inputGatheringTag'
+INPUT_PREPROCESSING_KEY = 'inputPreprocessingTag'
+ALGORITHM_EXECUTION_KEY = 'algorithmExecutionTag'
+
+# Values of SubmissionParameters
+LOWER_LEFT_LATITUDE = '-7.913'
+LOWER_LEFT_LONGITUDE = '-37.814'
+UPPER_RIGHT_LATITUDE = '-6.547'
+UPPER_RIGHT_LONGITUDE = '-35.757'
+INITIAL_DATE = '2014-06-12'
+FINAL_DATE = '2014-06-12'
+INPUT_GATHERING = 'Default'
+INPUT_PREPROCESSING = 'Default'
+ALGORITHM_EXECUTION = 'Default'
+
+# Keys of auth credentials
+USER_EMAIL_KEY = 'userEmail'
+USER_PASS_KEY = 'userPass'
+
+# Values of auth credentials
+ADMIN_EMAIL = 'admin@admin.com'
+ADMIN_PASSWORD = '4dm1n'
+
+# Successful feedback of submission processing
+SUBMIT_PROCESSING_SUCCESSFUL = "Tasks successfully added"
+
+
+def get_all_image_tasks(saps_instance_url):
+    """
+    Returns all ImageTasks stored in SAPS instance that had its URL specified.
+
+    :param saps_instance_url: URL of SAPS instance.
+    :return: List of all ImageTasks in SAPS instance's catalog.
+    """
+    get_all_image_tasks_url = saps_instance_url + PROCESSING_TASKS_URN
+    headers = {**get_admin_credentials()}
+    response = requests.get(url=get_all_image_tasks_url, headers=headers)
+    return response.json()
+
+
+def submit_processing(saps_instance_url):
+    """
+    Submits a processing with default SubmissionParameters to the SAPS instance
+    that had its URL specified.
+
+    :param saps_instance_url: URL of SAPS instance.
+    :return: Feedback of submission processing.
+    """
+    submit_processing_url = saps_instance_url + PROCESSING_TASKS_URN
+    data = {**get_admin_credentials(), **get_default_submission_parameters()}
+    response = requests.post(url=submit_processing_url, data=data)
+    return response.text
+
+
+def get_admin_credentials():
+    """
+    :return: admin credentials for both SAPS instances.
+    """
+    return {
+        USER_EMAIL_KEY: ADMIN_EMAIL,
+        USER_PASS_KEY: ADMIN_PASSWORD
+    }
+
+
+def get_default_submission_parameters():
+    """
+    :return: default SubmissionParameters.
+    """
+    return {
+        LOWER_LEFT_KEY: [LOWER_LEFT_LATITUDE, LOWER_LEFT_LONGITUDE],
+        UPPER_RIGHT_KEY: [UPPER_RIGHT_LATITUDE, UPPER_RIGHT_LONGITUDE],
+        INITIAL_DATE_KEY: INITIAL_DATE,
+        FINAL_DATE_KEY: FINAL_DATE,
+        INPUT_GATHERING_KEY: INPUT_GATHERING,
+        INPUT_PREPROCESSING_KEY: INPUT_PREPROCESSING,
+        ALGORITHM_EXECUTION_KEY: ALGORITHM_EXECUTION
+    }
+
+
+def main():
+    # Test preconditions
+    image_tasks_instance_1 = get_all_image_tasks(SAPS_INSTANCE_1_URL)
+    image_tasks_instance_2 = get_all_image_tasks(SAPS_INSTANCE_2_URL)
+    assert len(image_tasks_instance_1) == QTY_IMAGE_TASKS_INSTANCE_1
+    assert len(image_tasks_instance_2) == QTY_NO_IMAGE_TASKS
+
+    # Submits processing to Instance 2
+    assert submit_processing(SAPS_INSTANCE_2_URL) == SUBMIT_PROCESSING_SUCCESSFUL
+
+    # Instance 2 must have inserted each ImageTask stored in Instance 1
+    image_tasks_instance_1 = get_all_image_tasks(SAPS_INSTANCE_1_URL)
+    image_tasks_instance_2 = get_all_image_tasks(SAPS_INSTANCE_2_URL)
+    assert len(image_tasks_instance_1) == QTY_IMAGE_TASKS_INSTANCE_1
+    assert len(image_tasks_instance_2) == QTY_IMAGE_TASKS_INSTANCE_1
+
+    # Sorts lists of two instances to compare their elements as pairs
+    sorted(image_tasks_instance_1, key=lambda image_task: image_task[ID_ATTR_KEY])
+    sorted(image_tasks_instance_2, key=lambda image_task: image_task[ID_ATTR_KEY])
+
+    for i in range(QTY_IMAGE_TASKS_INSTANCE_1):
+        image_task_instance_1 = image_tasks_instance_1[i]
+        image_task_instance_2 = image_tasks_instance_2[i]
+
+        # Reused ImageTasks must have the 'remotely_archived' state
+        assert image_task_instance_1[STATE_ATTR_KEY] == ARCHIVED
+        assert image_task_instance_2[STATE_ATTR_KEY] == REMOTELY_ARCHIVED
+
+        # Except for the State, every other attribute of reused ImageTasks
+        # must be equal to processed ImageTasks, including the ID
+        for attr_key in list(image_task_instance_1.keys()):
+            if attr_key != STATE_ATTR_KEY:
+                assert image_task_instance_1[attr_key] == image_task_instance_2[attr_key]
+
+
+if __name__ == "__main__":
+    main()
+

src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java

@@ -9,6 +9,7 @@
 import java.util.Collection;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 
 public interface SubmissionDispatcher {
 
@@ -20,7 +21,8 @@ void addUserInDB(String userEmail, String userName, String userPass, boolean use
 	void addTaskNotificationIntoDB(String submissionId, String taskId, String userEmail)
 			throws SQLException;
 
-	List<Task> addTasks(SubmissionParameters submissionParameters, List<Date> processedDates);
+	List<Task> addTasks(SubmissionParameters submissionParameters,
+						Map<Date, List<ImageTask>> processedImageTasksGroupedByDate);
 
 	List<Task> addImageTasks(Collection<ImageTask> imageTasks);