From 01760359401d2b5cdaa60e2a3af36f1136925b17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Wed, 12 Jun 2019 14:54:20 -0300 Subject: [PATCH 01/21] Creates POJO to hold the information about file that was created by the processing of a ImageTask --- .../saps/engine/core/pojo/ImageTaskFile.java | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java new file mode 100644 index 000000000..6c1ecb47c --- /dev/null +++ b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java @@ -0,0 +1,39 @@ +package org.fogbowcloud.saps.engine.core.pojo; + +import org.fogbowcloud.saps.engine.core.model.ImageTask; + +/** + * Holds the information of a file generated by the processing of a {@link ImageTask}. + */ +public class ImageTaskFile { + + private String path; + private String name; + private String URL; + + public ImageTaskFile(String path, String name) { + this(path, name, null); + } + + public ImageTaskFile(String path, String name, String URL) { + this.path = path; + this.name = name; + this.URL = URL; + } + + public String getPath() { + return path; + } + + public String getName() { + return name; + } + + public String getURL() { + return URL; + } + + public void setURL(String URL) { + this.URL = URL; + } +} From c9ef4180129a00305eebe02c10632fc9c56cbb77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Wed, 12 Jun 2019 15:21:04 -0300 Subject: [PATCH 02/21] Extracts service to provide operations for communication with the Swift Object Store --- .../core/service/ObjectStoreService.java | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 src/main/java/org/fogbowcloud/saps/engine/core/service/ObjectStoreService.java diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/service/ObjectStoreService.java b/src/main/java/org/fogbowcloud/saps/engine/core/service/ObjectStoreService.java new file mode 100644 index 000000000..a90e653c3 --- /dev/null +++ b/src/main/java/org/fogbowcloud/saps/engine/core/service/ObjectStoreService.java @@ -0,0 +1,110 @@ +package org.fogbowcloud.saps.engine.core.service; + +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.apache.log4j.Logger; +import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.fogbowcloud.saps.engine.scheduler.util.SapsPropertiesConstants; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.*; + +/** + * Service that provides operations for communication with the Object Store. + */ +public class ObjectStoreService { + + private static final Logger LOGGER = Logger.getLogger(ObjectStoreService.class); + + private static final String HTTPS_SCHEME = "https"; + private static final String PROJECT_ID = "projectId"; + private static final String USER_ID = "userId"; + private static final String PASSWORD = "password"; + private static final String AUTH_URL = "authUrl"; + private static final String PATH_PARAM = "path"; + private static final String X_AUTH_TOKEN = "X-Auth-Token"; + private static final String HTTP_RESPONSE_SEPARATOR = "\n"; + private static final String ARCHIVER_PATH = "archiver/"; + private static final String DATA_OUTPUT_PATH = "/data/output/"; + + /** + * Returns all the paths for every file generated by the {@link ImageTask} + * that had its ID specified. + * + * @param properties Properties that contains Object Store information. + * @param imageTaskId ImageTask's ID. + * @return List of paths. + */ + public static List getImageTaskFilesPaths(Properties properties, String imageTaskId) { + try { + HttpClient client = HttpClients.createDefault(); + HttpGet httpget = prepareObjectStoreRequest(properties, imageTaskId); + HttpResponse response = client.execute(httpget); + return parseHttpResponse(response); + } catch (IOException | URISyntaxException e) { + LOGGER.error("", e); + return Collections.emptyList(); + } + + } + + /** + * Parses the given {@link HttpResponse} to a list of Strings. + * + * @param response Response to be parsed. + * @return List of Strings parsed from response. + * @throws IOException + */ + private static List parseHttpResponse(HttpResponse response) throws IOException { + return Arrays.asList(EntityUtils.toString(response.getEntity()).split(HTTP_RESPONSE_SEPARATOR)); + } + + /** + * Prepares a {@link HttpGet} request for the Object Store. + * + * @param properties Properties that contains Object Store information. + * @param imageTaskId ImageTask's ID. + * @return A {@link HttpGet} request. + * @throws URISyntaxException + */ + private static HttpGet prepareObjectStoreRequest(Properties properties, + String imageTaskId) throws URISyntaxException { + String objectStoreHost = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_HOST); + String objectStorePath = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_PATH); + String objectStoreContainer = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_CONTAINER); + String pathParamValue = ARCHIVER_PATH + imageTaskId + DATA_OUTPUT_PATH; + URI uri = new URIBuilder() + .setScheme(HTTPS_SCHEME) + .setHost(objectStoreHost) + .setPath(objectStorePath + "/" + objectStoreContainer) + .addParameter(PATH_PARAM, pathParamValue) + .build(); + LOGGER.debug("Getting list of files for task " + imageTaskId + " from " + uri); + HttpGet httpget = new HttpGet(uri); + Token token = getKeystoneToken(properties); + httpget.addHeader(X_AUTH_TOKEN, token.getAccessId()); + return httpget; + } + + /** + * Returns a Keystone Token for the given properties. + * + * @param properties Properties that contains Object Store information. + * @return Keystone Token. + */ + private static Token getKeystoneToken(Properties properties) { + Map credentials = new HashMap<>(); + credentials.put(PROJECT_ID, properties.getProperty(SapsPropertiesConstants.SWIFT_PROJECT_ID)); + credentials.put(USER_ID, properties.getProperty(SapsPropertiesConstants.SWIFT_USER_ID)); + credentials.put(PASSWORD, properties.getProperty(SapsPropertiesConstants.SWIFT_PASSWORD)); + credentials.put(AUTH_URL, properties.getProperty(SapsPropertiesConstants.SWIFT_AUTH_URL)); + KeystoneV3IdentityPlugin keystone = new KeystoneV3IdentityPlugin(properties); + return keystone.createToken(credentials); + } +} From e0cdd52d03921ae3fe0a1f3757a0a5b0cbdefe79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Wed, 12 Jun 2019 15:40:24 -0300 Subject: [PATCH 03/21] Extracts service to provide operations for processed ImageTasks --- .../core/service/ProcessedImagesService.java | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 src/main/java/org/fogbowcloud/saps/engine/core/service/ProcessedImagesService.java diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/service/ProcessedImagesService.java b/src/main/java/org/fogbowcloud/saps/engine/core/service/ProcessedImagesService.java new file mode 100644 index 000000000..67ca860e0 --- /dev/null +++ b/src/main/java/org/fogbowcloud/saps/engine/core/service/ProcessedImagesService.java @@ -0,0 +1,145 @@ +package org.fogbowcloud.saps.engine.core.service; + +import org.apache.log4j.Logger; +import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFile; +import org.fogbowcloud.saps.engine.scheduler.util.SapsPropertiesConstants; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.io.File; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.SignatureException; +import java.util.*; + +/** + * Service the provides operations with processed {@link ImageTask}. + */ +public class ProcessedImagesService { + + private static final Logger LOGGER = Logger.getLogger(ProcessedImagesService.class); + + private static final String TEMP_DIR_URL = "%s?temp_url_sig=%s&temp_url_expires=%s"; + private static final String UNAVAILABLE = "UNAVAILABLE"; + private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1"; + private static final String HTTPS_SCHEME = "https://"; + private static final String FILENAME_QUERY_PARAM = "&filename="; + private static final String GET = "GET"; + private static final String OBJECT_STORE_ENCRIPTION_FORMAT = "%s\n%s\n%s"; + + /** + * Generates a {@link ImageTaskFile} object for every file generated by the + * processing of {@link ImageTask} that had its ID specified. + * + * @param properties Properties that contains Object Store information. + * @param imageTaskId ImageTask's ID. + * @return List of {@link ImageTaskFile}. + */ + public static List generateImageTaskFiles(Properties properties, String imageTaskId) { + List filesPaths = ObjectStoreService.getImageTaskFilesPaths(properties, imageTaskId); + List imageTaskFiles = new ArrayList<>(); + for (String filePath: filesPaths) { + File file = new File(filePath); + ImageTaskFile imageTaskFile = new ImageTaskFile(filePath, file.getName()); + String imageTaskFileURL; + try { + imageTaskFileURL = generateTempURL(properties, imageTaskFile); + } catch (NoSuchAlgorithmException | SignatureException | InvalidKeyException e) { + LOGGER.error("Failed to generate download link for file " + filePath, e); + imageTaskFileURL = UNAVAILABLE; + } + imageTaskFile.setURL(imageTaskFileURL); + imageTaskFiles.add(imageTaskFile); + } + return imageTaskFiles; + } + + /** + * Generates a temporary access URL for given {@link ImageTaskFile}. + * + * @param properties Properties that contains Object Store information. + * @param imageTaskFile File to have its access URL generated. + * @return Temporary access URL for given {@link ImageTaskFile}. + */ + private static String generateTempURL(Properties properties, ImageTaskFile imageTaskFile) + throws NoSuchAlgorithmException, SignatureException, InvalidKeyException { + String objectStoreHost = properties + .getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_HOST); + String objectStorePath = properties + .getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_PATH); + String objectStoreContainer = properties + .getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_CONTAINER); + String objectStoreKey = properties + .getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_KEY); + String path = generateTempURLPath( + objectStorePath, + objectStoreContainer, + imageTaskFile.getPath(), + objectStoreKey); + return HTTPS_SCHEME + + objectStoreHost + + path + + FILENAME_QUERY_PARAM + + imageTaskFile.getName(); + } + + /** + * Generates the path of a temporary URL. + * + * @param swiftPath Path to Swift Object Store. + * @param container Container of Swift Object Store. + * @param filePath Path to the file that will be accessed. + * @param key Key of encryption. + * @return Path of a temporary URL. + */ + private static String generateTempURLPath( + String swiftPath, + String container, + String filePath, + String key) + throws NoSuchAlgorithmException, SignatureException, InvalidKeyException { + Formatter objectStoreFormatter = new Formatter(); + String path = swiftPath + "/" + container + "/" + filePath; + objectStoreFormatter.format(OBJECT_STORE_ENCRIPTION_FORMAT, GET, Long.MAX_VALUE, path); + String signature = calculateRFC2104HMAC(objectStoreFormatter.toString(), key); + objectStoreFormatter.close(); + objectStoreFormatter = new Formatter(); + objectStoreFormatter.format(TEMP_DIR_URL, path, signature, Long.MAX_VALUE); + String tempURLPath = objectStoreFormatter.toString(); + objectStoreFormatter.close(); + return tempURLPath; + } + + /** + * Encrypts the given data with the given key. + * + * @param data Data to be encrypted. + * @param key Key of encryption. + * @return The encrypted string. + */ + private static String calculateRFC2104HMAC(String data, String key) + throws SignatureException, NoSuchAlgorithmException, InvalidKeyException { + SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), HMAC_SHA1_ALGORITHM); + Mac mac = Mac.getInstance(HMAC_SHA1_ALGORITHM); + mac.init(signingKey); + return parseToHexString(mac.doFinal(data.getBytes())); + } + + /** + * Parses a array of bytes to a hexadecimal string. + * + * @param bytes Array of bytes to be parsed. + * @return Hexadecimal string parsed from given array of bytes. + */ + private static String parseToHexString(byte[] bytes) { + Formatter formatter = new Formatter(); + for (byte b : bytes) { + formatter.format("%02x", b); + } + String hexString = formatter.toString(); + formatter.close(); + return hexString; + } + +} From 021743f2a345ce7d7aab6bb829ad6ef01a1427b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Wed, 12 Jun 2019 22:40:41 -0300 Subject: [PATCH 04/21] Removes duplicated code from ProcessedImagesEmailBuilder --- .../util/ProcessedImagesEmailBuilder.java | 240 ++++-------------- 1 file changed, 44 insertions(+), 196 deletions(-) diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java index 2e9859d8f..75f3a8c32 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java @@ -1,47 +1,26 @@ package org.fogbowcloud.saps.engine.scheduler.util; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; import org.apache.log4j.Logger; -import org.fogbowcloud.manager.core.plugins.identity.openstack.KeystoneV3IdentityPlugin; -import org.fogbowcloud.manager.occi.model.Token; import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFile; +import org.fogbowcloud.saps.engine.core.service.ProcessedImagesService; import org.fogbowcloud.saps.engine.scheduler.restlet.DatabaseApplication; import org.fogbowcloud.saps.notifier.GoogleMail; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; import javax.mail.MessagingException; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; -import java.security.SignatureException; import java.sql.SQLException; -import java.util.*; +import java.util.List; +import java.util.Properties; public class ProcessedImagesEmailBuilder implements Runnable { private static final Logger LOGGER = Logger.getLogger(ProcessedImagesEmailBuilder.class); - private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1"; private static final String UNAVAILABLE = "UNAVAILABLE"; - private static final String TEMP_DIR_URL = "%s?temp_url_sig=%s&temp_url_expires=%s"; - - private static final String PROJECT_ID = "projectId"; - private static final String PASSWORD = "password"; - private static final String USER_ID = "userId"; - private static final String AUTH_URL = "authUrl"; private static final String TASK_ID = "taskId"; private static final String REGION = "region"; private static final String COLLECTION_TIER_NAME = "collectionTierName"; @@ -54,18 +33,19 @@ public class ProcessedImagesEmailBuilder implements Runnable { private DatabaseApplication application; private Properties properties; private String userEmail; - private List images; + private List imageTasksIds; + private StringBuilder error; public ProcessedImagesEmailBuilder( DatabaseApplication databaseApplication, Properties properties, String userEmail, - List images - ) { + List imageTasksIds) { this.application = databaseApplication; this.properties = properties; this.userEmail = userEmail; - this.images = images; + this.imageTasksIds = imageTasksIds; + this.error = new StringBuilder(); } @Override @@ -74,67 +54,34 @@ public void run() { builder.append("Creating email for user "); builder.append(userEmail); builder.append(" with images:\n"); - for (String str: images) { - builder.append(str).append("\n"); + for (String imageTaskId: imageTasksIds) { + builder.append(imageTaskId).append("\n"); } LOGGER.info(builder.toString()); - - StringBuilder errorBuilder = new StringBuilder(); - JSONArray tasklist = generateAllTasksJsons(errorBuilder); - sendTaskEmail(errorBuilder, tasklist); - sendErrorEmail(errorBuilder); + JSONArray tasklist = generateAllTasksJsons(); + sendTaskEmail(tasklist); + sendErrorEmail(); } - JSONArray generateAllTasksJsons(StringBuilder errorBuilder) { + JSONArray generateAllTasksJsons() { JSONArray tasklist = new JSONArray(); - for (String str: images) { + for (String imageTaskId: imageTasksIds) { try { - tasklist.put(generateTaskEmailJson(properties, str)); - } catch (IOException | URISyntaxException | IllegalArgumentException e) { - LOGGER.error("Failed to fetch image from object store.", e); - errorBuilder - .append("Failed to fetch image from object store.").append("\n") - .append(ExceptionUtils.getStackTrace(e)).append("\n"); - try { - JSONObject task = new JSONObject(); - task.put(TASK_ID, str); - task.put(STATUS, UNAVAILABLE); - tasklist.put(task); - } catch (JSONException e1) { - LOGGER.error("Failed to create UNAVAILABLE task json.", e); - } + tasklist.put(generateTaskEmailJson(properties, imageTaskId)); } catch (SQLException e) { LOGGER.error("Failed to fetch image from database.", e); - errorBuilder - .append("Failed to fetch image from database.").append("\n") + error.append("Failed to fetch image from database.").append("\n") .append(ExceptionUtils.getStackTrace(e)).append("\n"); - try { - JSONObject task = new JSONObject(); - task.put(TASK_ID, str); - task.put(STATUS, UNAVAILABLE); - tasklist.put(task); - } catch (JSONException e1) { - LOGGER.error("Failed to create UNAVAILABLE task json.", e); - } } catch (JSONException e) { LOGGER.error("Failed to create task json.", e); - errorBuilder - .append("Failed to create task json.").append("\n") + error.append("Failed to create task json.").append("\n") .append(ExceptionUtils.getStackTrace(e)).append("\n"); - try { - JSONObject task = new JSONObject(); - task.put(TASK_ID, str); - task.put(STATUS, UNAVAILABLE); - tasklist.put(task); - } catch (JSONException e1) { - LOGGER.error("Failed to create UNAVAILABLE task json.", e); - } } } return tasklist; } - private void sendTaskEmail(StringBuilder errorBuilder, JSONArray tasklist) { + private void sendTaskEmail(JSONArray tasklist) { try { GoogleMail.Send( properties.getProperty(SapsPropertiesConstants.NO_REPLY_EMAIL), @@ -145,21 +92,21 @@ private void sendTaskEmail(StringBuilder errorBuilder, JSONArray tasklist) { ); } catch (MessagingException | JSONException e) { LOGGER.error("Failed to send email with images download links.", e); - errorBuilder + error .append("Failed to send email with images download links.").append("\n") .append(ExceptionUtils.getStackTrace(e)).append("\n"); } } - private void sendErrorEmail(StringBuilder errorBuilder) { - if (!errorBuilder.toString().isEmpty()) { + private void sendErrorEmail() { + if (!error.toString().isEmpty()) { try { GoogleMail.Send( properties.getProperty(SapsPropertiesConstants.NO_REPLY_EMAIL), properties.getProperty(SapsPropertiesConstants.NO_REPLY_PASS), "sebal.no.reply@gmail.com", "[SAPS] Errors during image temporary link creation", - errorBuilder.toString() + error.toString() ); } catch (MessagingException e) { LOGGER.error("Failed to send email with errors to admins.", e); @@ -167,131 +114,32 @@ private void sendErrorEmail(StringBuilder errorBuilder) { } } - private String toHexString(byte[] bytes) { - Formatter formatter = new Formatter(); - - for (byte b : bytes) { - formatter.format("%02x", b); - } - - String hexString = formatter.toString(); - formatter.close(); - - return hexString; - } - - private String calculateRFC2104HMAC(String data, String key) - throws SignatureException, NoSuchAlgorithmException, InvalidKeyException - { - SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), HMAC_SHA1_ALGORITHM); - Mac mac = Mac.getInstance(HMAC_SHA1_ALGORITHM); - mac.init(signingKey); - return toHexString(mac.doFinal(data.getBytes())); - } - - String generateTempURL(String swiftPath, String container, String filePath, String key) - throws NoSuchAlgorithmException, SignatureException, InvalidKeyException { - String path = swiftPath + "/" + container + "/" + filePath; - - Formatter objectStoreFormatter = new Formatter(); - objectStoreFormatter.format("%s\n%s\n%s", "GET", Long.MAX_VALUE, path); - String signature = calculateRFC2104HMAC(objectStoreFormatter.toString(), key); - objectStoreFormatter.close(); - - objectStoreFormatter = new Formatter(); - objectStoreFormatter.format( - TEMP_DIR_URL, - path, - signature, - Long.MAX_VALUE - ); - String res = objectStoreFormatter.toString(); - objectStoreFormatter.close(); - - return res; - } - - JSONObject generateTaskEmailJson(Properties properties, String imageid) - throws URISyntaxException, IOException, SQLException, JSONException { + JSONObject generateTaskEmailJson(Properties properties, String imageTaskId) + throws SQLException, JSONException { JSONObject result = new JSONObject(); + result.put(TASK_ID, imageTaskId); - String objectStoreHost = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_HOST); - String objectStorePath = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_PATH); - String objectStoreContainer = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_CONTAINER); - String objectStoreKey = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_KEY); - - ImageTask task = application.getTask(imageid); - - List files = getTaskFilesFromObjectStore( - properties, objectStoreHost, objectStorePath, objectStoreContainer, task - ); - - result.put(TASK_ID, task.getTaskId()); - result.put(REGION, task.getRegion()); - result.put(COLLECTION_TIER_NAME, task.getCollectionTierName()); - result.put(IMAGE_DATE, task.getImageDate()); - - JSONArray filelist = new JSONArray(); - for (String str: files) { - File f = new File(str); - String fileName = f.getName(); - try { - JSONObject fileobject = new JSONObject(); - fileobject.put(NAME, fileName); - fileobject.put(URL, "https://" + objectStoreHost + generateTempURL( - objectStorePath, - objectStoreContainer, - str, - objectStoreKey) + "&filename=" + fileName - ); - filelist.put(fileobject); - } catch (NoSuchAlgorithmException | SignatureException | InvalidKeyException e) { - LOGGER.error("Failed to generate download link for file " + str, e); - try { - JSONObject fileobject = new JSONObject(); - fileobject.put(NAME, fileName); - fileobject.put(URL, UNAVAILABLE); - filelist.put(fileobject); - } catch (JSONException e1) { - LOGGER.error("Failed to create UNAVAILABLE temp url json.", e); - } + try { + ImageTask task = application.getTask(imageTaskId); + result.put(REGION, task.getRegion()); + result.put(COLLECTION_TIER_NAME, task.getCollectionTierName()); + result.put(IMAGE_DATE, task.getImageDate()); + List imageTaskFiles = ProcessedImagesService + .generateImageTaskFiles(properties, imageTaskId); + JSONArray filesJSONArray = new JSONArray(); + for (ImageTaskFile imageTaskFile: imageTaskFiles) { + JSONObject imageTaskFileJSONObject = new JSONObject(); + imageTaskFileJSONObject.put(NAME, imageTaskFile.getName()); + imageTaskFileJSONObject.put(URL, imageTaskFile.getURL()); + filesJSONArray.put(imageTaskFileJSONObject); } + result.put(FILES, filesJSONArray); + } catch (SQLException | JSONException e) { + result.put(STATUS, UNAVAILABLE); + throw e; } - result.put(FILES, filelist); return result; } - List getTaskFilesFromObjectStore(Properties properties, String objectStoreHost, String objectStorePath, String objectStoreContainer, ImageTask task) throws URISyntaxException, IOException { - Token token = getKeystoneToken(properties); - - HttpClient client = HttpClients.createDefault(); - HttpGet httpget = prepObjectStoreRequest(objectStoreHost, objectStorePath, objectStoreContainer, task, token); - HttpResponse response = client.execute(httpget); - - return Arrays.asList(EntityUtils.toString(response.getEntity()).split("\n")); - } - - private HttpGet prepObjectStoreRequest(String objectStoreHost, String objectStorePath, String objectStoreContainer, ImageTask task, Token token) throws URISyntaxException { - URI uri = new URIBuilder() - .setScheme("https") - .setHost(objectStoreHost) - .setPath(objectStorePath + "/" + objectStoreContainer) - .addParameter("path", "archiver/" + task.getTaskId() + "/data/output/") - .build(); - LOGGER.debug("Getting list of files for task " + task.getTaskId() + " from " + uri); - HttpGet httpget = new HttpGet(uri); - httpget.addHeader("X-Auth-Token", token.getAccessId()); - return httpget; - } - - private Token getKeystoneToken(Properties properties) { - KeystoneV3IdentityPlugin keystone = new KeystoneV3IdentityPlugin(properties); - Map credentials = new HashMap<>(); - credentials.put(AUTH_URL, properties.getProperty(SapsPropertiesConstants.SWIFT_AUTH_URL)); - credentials.put(PROJECT_ID, properties.getProperty(SapsPropertiesConstants.SWIFT_PROJECT_ID)); - credentials.put(USER_ID, properties.getProperty(SapsPropertiesConstants.SWIFT_USER_ID)); - credentials.put(PASSWORD, properties.getProperty(SapsPropertiesConstants.SWIFT_PASSWORD)); - return keystone.createToken(credentials); - } } From 0b794a6c731ce440768a757007bf400fbe8567b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Thu, 13 Jun 2019 00:20:08 -0300 Subject: [PATCH 05/21] Creates POJO to represent a list of ImageTaskFile --- .../engine/core/pojo/ImageTaskFileList.java | 54 +++++++++++++++++++ .../core/service/ProcessedImagesService.java | 17 +++--- 2 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java new file mode 100644 index 000000000..06d44ba21 --- /dev/null +++ b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java @@ -0,0 +1,54 @@ +package org.fogbowcloud.saps.engine.core.pojo; + +import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.util.List; + +/** + * Represents a list of {@link ImageTaskFile} for a specific {@link ImageTask}. + */ +public class ImageTaskFileList { + + private static final String IMAGE_TASK_ID = "imageTaskId"; + private static final String REGION = "region"; + private static final String COLLECTION_TIER_NAME = "collectionTierName"; + private static final String IMAGE_DATE = "imageDate"; + private static final String NAME = "name"; + private static final String URL = "url"; + private static final String FILES = "files"; + private static final String STATUS = "status"; + private static final String UNAVAILABLE = "UNAVAILABLE"; + + private ImageTask imageTask; + private List imageTaskFiles; + + public ImageTaskFileList(ImageTask imageTask, List imageTaskFiles) { + this.imageTask = imageTask; + this.imageTaskFiles = imageTaskFiles; + } + + public JSONObject toJSON() throws JSONException { + JSONObject jsonObject = new JSONObject(); + jsonObject.put(IMAGE_TASK_ID, imageTask.getTaskId()); + try { + jsonObject.put(REGION, imageTask.getRegion()); + jsonObject.put(COLLECTION_TIER_NAME, imageTask.getCollectionTierName()); + jsonObject.put(IMAGE_DATE, imageTask.getImageDate()); + JSONArray filesJSONArray = new JSONArray(); + for (ImageTaskFile imageTaskFile : imageTaskFiles) { + JSONObject imageTaskFileJSONObject = new JSONObject(); + imageTaskFileJSONObject.put(NAME, imageTaskFile.getName()); + imageTaskFileJSONObject.put(URL, imageTaskFile.getURL()); + filesJSONArray.put(imageTaskFileJSONObject); + } + jsonObject.put(FILES, filesJSONArray); + } catch (JSONException e) { + jsonObject.put(STATUS, UNAVAILABLE); + throw e; + } + return jsonObject; + } +} diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/service/ProcessedImagesService.java b/src/main/java/org/fogbowcloud/saps/engine/core/service/ProcessedImagesService.java index 67ca860e0..9ecff7732 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/service/ProcessedImagesService.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/service/ProcessedImagesService.java @@ -3,6 +3,7 @@ import org.apache.log4j.Logger; import org.fogbowcloud.saps.engine.core.model.ImageTask; import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFile; +import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFileList; import org.fogbowcloud.saps.engine.scheduler.util.SapsPropertiesConstants; import javax.crypto.Mac; @@ -29,15 +30,15 @@ public class ProcessedImagesService { private static final String OBJECT_STORE_ENCRIPTION_FORMAT = "%s\n%s\n%s"; /** - * Generates a {@link ImageTaskFile} object for every file generated by the - * processing of {@link ImageTask} that had its ID specified. + * Generates a {@link ImageTaskFileList} object containing every file generated + * by the processing of {@link ImageTask} that was specified. * - * @param properties Properties that contains Object Store information. - * @param imageTaskId ImageTask's ID. - * @return List of {@link ImageTaskFile}. + * @param properties Properties that contains Object Store information. + * @param imageTask ImageTask that will have its files returned. + * @return A {@link ImageTaskFileList}. */ - public static List generateImageTaskFiles(Properties properties, String imageTaskId) { - List filesPaths = ObjectStoreService.getImageTaskFilesPaths(properties, imageTaskId); + public static ImageTaskFileList generateImageTaskFiles(Properties properties, ImageTask imageTask) { + List filesPaths = ObjectStoreService.getImageTaskFilesPaths(properties, imageTask.getTaskId()); List imageTaskFiles = new ArrayList<>(); for (String filePath: filesPaths) { File file = new File(filePath); @@ -52,7 +53,7 @@ public static List generateImageTaskFiles(Properties properties, imageTaskFile.setURL(imageTaskFileURL); imageTaskFiles.add(imageTaskFile); } - return imageTaskFiles; + return new ImageTaskFileList(imageTask, imageTaskFiles); } /** From 70e7f4262f77e6cbf1807fa1c8077644f6bca8af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Thu, 13 Jun 2019 00:25:45 -0300 Subject: [PATCH 06/21] Init renaming of resources to better understanding, separating a resource for processed ImageTasks and another resource for email --- .../saps/engine/scheduler/restlet/DatabaseApplication.java | 3 +-- ...{ProcessedImagesResource.java => ImagesEmailResource.java} | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) rename src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/{ProcessedImagesResource.java => ImagesEmailResource.java} (91%) diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java index 50e43c633..d380e5571 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java @@ -1,7 +1,6 @@ package org.fogbowcloud.saps.engine.scheduler.restlet; import java.io.File; -import java.io.IOException; import java.sql.SQLException; import java.text.ParseException; import java.util.Collections; @@ -95,7 +94,7 @@ public Restlet createInboundRoot() { router.attach("/images/{imgName}", ImageResource.class); router.attach("/regions/details", RegionResource.class); router.attach("/regions/search", RegionResource.class); - router.attach("/email", ProcessedImagesResource.class); + router.attach("/email", ImagesEmailResource.class); router.attach("/archivedTasks", ArchivedTasksResource.class); return router; diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ProcessedImagesResource.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java similarity index 91% rename from src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ProcessedImagesResource.java rename to src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java index f424e4345..1d2e0d470 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ProcessedImagesResource.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java @@ -13,9 +13,9 @@ import java.util.Arrays; import java.util.Properties; -public class ProcessedImagesResource extends BaseResource { +public class ImagesEmailResource extends BaseResource { - public static final Logger LOGGER = Logger.getLogger(ProcessedImagesResource.class); + public static final Logger LOGGER = Logger.getLogger(ImagesEmailResource.class); private static final String REQUEST_ATTR_PROCESSED_IMAGES = "images_id[]"; From d846927cd14ca613b854c5c73e5add97187174a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Thu, 13 Jun 2019 16:27:21 -0300 Subject: [PATCH 07/21] Creates method to return from Catalog a list of ImageTasks given a list of IDs --- .../engine/core/database/ImageDataStore.java | 2 ++ .../core/database/JDBCImageDataStore.java | 27 ++++++++++++++----- .../core/dispatcher/SubmissionDispatcher.java | 2 ++ .../dispatcher/SubmissionDispatcherImpl.java | 5 ++++ .../restlet/DatabaseApplication.java | 4 +++ 5 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/database/ImageDataStore.java b/src/main/java/org/fogbowcloud/saps/engine/core/database/ImageDataStore.java index bf3544325..af57b01a2 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/database/ImageDataStore.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/database/ImageDataStore.java @@ -126,4 +126,6 @@ public List getProcessedImages( String inputGathering, String inputPreprocessing, String algorithmExecution) throws SQLException; + + List getImageTasks(String[] imageTasksIds) throws SQLException; } diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/database/JDBCImageDataStore.java b/src/main/java/org/fogbowcloud/saps/engine/core/database/JDBCImageDataStore.java index aff7d1f4f..8bc0e4412 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/database/JDBCImageDataStore.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/database/JDBCImageDataStore.java @@ -6,12 +6,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.dbcp2.BasicDataSource; @@ -1020,6 +1015,26 @@ public List getAllTasks() throws SQLException { } } + private static final String SELECT_IMAGES_IN_ID_LIST_SQL = "SELECT * FROM " + IMAGE_TABLE_NAME + + " WHERE " + TASK_ID_COL + " IN ?"; + + @Override + public List getImageTasks(String[] imageTasksIds) throws SQLException { + PreparedStatement statement = null; + Connection conn = null; + try { + conn = getConnection(); + statement = conn.prepareStatement(SELECT_IMAGES_IN_ID_LIST_SQL); + statement.setString(1, Arrays.toString(imageTasksIds)); + statement.setQueryTimeout(300); + statement.execute(); + ResultSet rs = statement.getResultSet(); + return extractImageTaskFrom(rs); + } finally { + close(statement, conn); + } + } + private static final String SELECT_USER_SQL = "SELECT * FROM " + USERS_TABLE_NAME + " WHERE " + USER_EMAIL_COL + " = ?"; diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java index 4e18161cf..f90771223 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java @@ -41,4 +41,6 @@ void removeUserNotification(String submissionId, String taskId, String userEmail boolean isUserNotifiable(String userEmail) throws SQLException; List searchProcessedTasks(SubmissionParameters submissionParameters); + + List getImageTasks(String[] imageTasksIds) throws SQLException; } diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherImpl.java b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherImpl.java index a189c57f8..5f59886b6 100755 --- a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherImpl.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherImpl.java @@ -269,6 +269,11 @@ public List getTaskListInDB() throws SQLException, ParseException { return imageStore.getAllTasks(); } + @Override + public List getImageTasks(String[] imageTasksIds) throws SQLException { + return imageStore.getImageTasks(imageTasksIds); + } + @Override public List getUsersToNotify() throws SQLException { List wards = imageStore.getUsersToNotify(); diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java index d380e5571..e2e6328bf 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java @@ -103,6 +103,10 @@ public Restlet createInboundRoot() { public List getTasks() throws SQLException, ParseException { return submissionDispatcher.getTaskListInDB(); } + + public List getImageTasks(String[] imageTasksIds) throws SQLException { + return submissionDispatcher.getImageTasks(imageTasksIds); + } public List getTasksInState(ImageTaskState imageState) throws SQLException { return this.submissionDispatcher.getTasksInState(imageState); From 2ce7dbac6e776aaf2545cd352d0030e585522cf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Thu, 13 Jun 2019 17:11:45 -0300 Subject: [PATCH 08/21] Creates api route to return the files of processed image tasks that have had its ids specified --- .../resource/ArchivedTasksResource.java | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java index 5e6318471..95836cf52 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java @@ -1,16 +1,26 @@ package org.fogbowcloud.saps.engine.scheduler.restlet.resource; +import java.sql.SQLException; +import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import org.apache.log4j.Logger; import org.fogbowcloud.saps.engine.core.dispatcher.SubmissionParameters; import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFileList; +import org.fogbowcloud.saps.engine.core.service.ProcessedImagesService; +import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import org.restlet.data.Form; +import org.restlet.data.Header; import org.restlet.data.MediaType; import org.restlet.representation.Representation; import org.restlet.representation.StringRepresentation; +import org.restlet.resource.Get; import org.restlet.resource.Post; +import org.restlet.util.Series; /** * Responsable for retrieve information about already processed - archived - tasks. @@ -19,6 +29,8 @@ public class ArchivedTasksResource extends BaseResource { private static final Logger LOGGER = Logger.getLogger(ArchivedTasksResource.class); + private static final String REQUEST_ATTR_PROCESSED_IMAGES = "images_id[]"; + public ArchivedTasksResource() { super(); } @@ -49,4 +61,35 @@ public Representation getProcessedTasksInInterval(Representation representation) return new StringRepresentation(resObj.toString(), MediaType.APPLICATION_JSON); } + /** + * Returns the files generated by execution of each {@link ImageTask} that + * had its ID specified. + */ + @SuppressWarnings("unchecked") + @Get + public Representation getProcessedImageTasksFiles() { + Series
series = (Series
) getRequestAttributes() + .get("org.restlet.http.headers"); + + String[] imageTasksIds = series.getValuesArray(REQUEST_ATTR_PROCESSED_IMAGES, true); + LOGGER.info("Recovering files from processed ImageTasks from list of IDs: " + + Arrays.toString(imageTasksIds)); + + JSONArray resultJSONArray = new JSONArray(); + try { + List imageTasks = application.getImageTasks(imageTasksIds); + List imageTaskFileLists = imageTasks.stream() + .map(imageTask -> ProcessedImagesService.generateImageTaskFiles( + application.getProperties(), + imageTask)) + .collect(Collectors.toList()); + for (ImageTaskFileList imageTaskFileList: imageTaskFileLists) { + resultJSONArray.put(imageTaskFileList.toJSON()); + } + } catch (SQLException | JSONException e) { + LOGGER.error("Error while returning files of processed ImageTasks", e); + } + return new StringRepresentation(resultJSONArray.toString(), MediaType.APPLICATION_JSON); + } + } \ No newline at end of file From fe3890417d71290c602e05ebcf59e7dd2b931e63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Fri, 14 Jun 2019 00:58:59 -0300 Subject: [PATCH 09/21] Extracts methods in ImageTaskFiles retrieval --- .../resource/ArchivedTasksResource.java | 58 +++++++++++++------ 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java index 95836cf52..463604f24 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java @@ -1,6 +1,7 @@ package org.fogbowcloud.saps.engine.scheduler.restlet.resource; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -23,7 +24,7 @@ import org.restlet.util.Series; /** - * Responsable for retrieve information about already processed - archived - tasks. + * Responsible for retrieve information about already processed {@link ImageTask}. */ public class ArchivedTasksResource extends BaseResource { @@ -31,10 +32,6 @@ public class ArchivedTasksResource extends BaseResource { private static final String REQUEST_ATTR_PROCESSED_IMAGES = "images_id[]"; - public ArchivedTasksResource() { - super(); - } - /** * Returns all processed tasks that matches the execution parameters passed * in {@code representation}. @@ -75,21 +72,48 @@ public Representation getProcessedImageTasksFiles() { LOGGER.info("Recovering files from processed ImageTasks from list of IDs: " + Arrays.toString(imageTasksIds)); - JSONArray resultJSONArray = new JSONArray(); - try { - List imageTasks = application.getImageTasks(imageTasksIds); - List imageTaskFileLists = imageTasks.stream() - .map(imageTask -> ProcessedImagesService.generateImageTaskFiles( - application.getProperties(), - imageTask)) - .collect(Collectors.toList()); + List imageTaskFileLists = getImageTaskFileLists(imageTasksIds); + JSONArray imageTaskFileListsJSONArray = parseToJsonArray(imageTaskFileLists); + return new StringRepresentation(imageTaskFileListsJSONArray.toString(), MediaType.APPLICATION_JSON); + } + + /** + * Gets a list of {@link ImageTaskFileList} given the IDs of {@link ImageTask}. + * + * @param imageTasksIds IDs of ImageTasks to have its ImageTaskFileList returned. + * @return List of {@link ImageTaskFileList}. + */ + private List getImageTaskFileLists(String[] imageTasksIds) { + List imageTaskFileLists = new ArrayList<>(); + try { + List imageTasks = application.getImageTasks(imageTasksIds); + imageTaskFileLists = imageTasks.stream() + .map(imageTask -> ProcessedImagesService.generateImageTaskFiles( + application.getProperties(), + imageTask)) + .collect(Collectors.toList()); + } catch (SQLException e) { + LOGGER.error("Error while getting ImageTasks from Catalogue", e); + } + return imageTaskFileLists; + } + + /** + * Parses the specified list of {@link ImageTaskFileList} to a {@link JSONArray}. + * + * @param imageTaskFileLists List of ImageTaskFileList to be parsed. + * @return List of ImageTaskFileList parsed to JSONArray. + */ + private JSONArray parseToJsonArray(List imageTaskFileLists) { + JSONArray resultJSONArray = new JSONArray(); + try { for (ImageTaskFileList imageTaskFileList: imageTaskFileLists) { resultJSONArray.put(imageTaskFileList.toJSON()); } - } catch (SQLException | JSONException e) { - LOGGER.error("Error while returning files of processed ImageTasks", e); + } catch (JSONException e) { + LOGGER.error("Error while parsing list of ImageTaskFileList to JSONArray", e); } - return new StringRepresentation(resultJSONArray.toString(), MediaType.APPLICATION_JSON); - } + return resultJSONArray; + } } \ No newline at end of file From c4c466d3dab7b703bb8536d80438e0b913a70058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Fri, 14 Jun 2019 11:38:27 -0300 Subject: [PATCH 10/21] Creates Service to handle email operations --- .../engine/core/service/EmailService.java | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 src/main/java/org/fogbowcloud/saps/engine/core/service/EmailService.java diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/service/EmailService.java b/src/main/java/org/fogbowcloud/saps/engine/core/service/EmailService.java new file mode 100644 index 000000000..5a472c0e0 --- /dev/null +++ b/src/main/java/org/fogbowcloud/saps/engine/core/service/EmailService.java @@ -0,0 +1,59 @@ +package org.fogbowcloud.saps.engine.core.service; + +import org.fogbowcloud.saps.engine.scheduler.util.SapsPropertiesConstants; +import org.fogbowcloud.saps.notifier.GoogleMail; + +import java.util.Properties; + +/** + * Service to handle Email operations. + */ +public class EmailService { + + /** + * Asynchronously sends an email with specified parameters and default + * no-reply email and password. + * + * @param recipientEmail Recipient email. + * @param title Email title. + * @param message Email message. + */ + public static void sendEmail( + Properties properties, + String recipientEmail, + String title, + String message) { + sendEmail( + properties.getProperty(SapsPropertiesConstants.NO_REPLY_EMAIL), + properties.getProperty(SapsPropertiesConstants.NO_REPLY_PASS), + recipientEmail, + title, + message + ); + } + + /** + * Asynchronously sends an email with specified parameters. + * + * @param username Username of sender account. + * @param password Password of sender account. + * @param recipientEmail Recipient email. + * @param title Email title. + * @param message Email message. + */ + public static void sendEmail( + String username, + String password, + String recipientEmail, + String title, + String message) { + Thread emailThread = new Thread(() -> GoogleMail.Send( + username, + password, + recipientEmail, + title, + message + )); + emailThread.start(); + } +} From fd4dbc59be64b760a46df9e87b08a104b4a5aaa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Fri, 14 Jun 2019 11:39:40 -0300 Subject: [PATCH 11/21] Creates reduced constructor for ImageTask --- .../org/fogbowcloud/saps/engine/core/model/ImageTask.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/model/ImageTask.java b/src/main/java/org/fogbowcloud/saps/engine/core/model/ImageTask.java index 22f13675a..d4a0739b4 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/model/ImageTask.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/model/ImageTask.java @@ -89,7 +89,13 @@ public ImageTask(JSONObject imageTaskJsonObject) throws JSONException { ); } - public String getTaskId() { + public ImageTask(String taskId, String region, Date imageDate) { + this.taskId = taskId; + this.region = region; + this.imageDate = imageDate; + } + + public String getTaskId() { return taskId; } From e4c46249a3792ae292a049d95d612b5fca6004ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Fri, 14 Jun 2019 11:40:13 -0300 Subject: [PATCH 12/21] Creates constructor from JSONObject for ImageTaskFileList --- .../engine/core/pojo/ImageTaskFileList.java | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java index 06d44ba21..2332a9278 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java @@ -5,6 +5,8 @@ import org.json.JSONException; import org.json.JSONObject; +import java.util.ArrayList; +import java.util.Date; import java.util.List; /** @@ -30,13 +32,32 @@ public ImageTaskFileList(ImageTask imageTask, List imageTaskFiles this.imageTaskFiles = imageTaskFiles; } + public ImageTaskFileList(JSONObject jsonObject) throws JSONException { + ImageTask imageTask = new ImageTask(jsonObject.getString(IMAGE_TASK_ID), + jsonObject.getString(REGION), + (Date) jsonObject.get(IMAGE_DATE) + ); + List imageTaskFiles = new ArrayList<>(); + JSONArray filesJSONArray = jsonObject.getJSONArray(FILES); + for (int i = 0; i < filesJSONArray.length(); i++) { + JSONObject imageTaskFileJSON = filesJSONArray.optJSONObject(i); + imageTaskFiles.add(new ImageTaskFile( + null, + imageTaskFileJSON.getString(NAME), + imageTaskFileJSON.getString(URL) + )); + } + this.imageTask = imageTask; + this.imageTaskFiles = imageTaskFiles; + } + public JSONObject toJSON() throws JSONException { JSONObject jsonObject = new JSONObject(); jsonObject.put(IMAGE_TASK_ID, imageTask.getTaskId()); try { jsonObject.put(REGION, imageTask.getRegion()); - jsonObject.put(COLLECTION_TIER_NAME, imageTask.getCollectionTierName()); jsonObject.put(IMAGE_DATE, imageTask.getImageDate()); + jsonObject.put(COLLECTION_TIER_NAME, imageTask.getCollectionTierName()); JSONArray filesJSONArray = new JSONArray(); for (ImageTaskFile imageTaskFile : imageTaskFiles) { JSONObject imageTaskFileJSONObject = new JSONObject(); From 0ac4ea052cfbfc4345efdaf12154f3a9124022c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Fri, 14 Jun 2019 14:01:49 -0300 Subject: [PATCH 13/21] Replaces ProcessedImagesEmailBuilder by EmailService and refacts ImagesEmailResource --- .../restlet/resource/ImagesEmailResource.java | 222 +++++++++++++- .../util/ProcessedImagesEmailBuilder.java | 145 --------- .../util/TestProcessedImagesEmailBuilder.java | 280 ------------------ 3 files changed, 206 insertions(+), 441 deletions(-) delete mode 100644 src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java delete mode 100644 src/test/java/org/fogbowcloud/saps/engine/scheduler/util/TestProcessedImagesEmailBuilder.java diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java index 1d2e0d470..4b61c20f0 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java @@ -2,46 +2,236 @@ import org.apache.commons.httpclient.HttpStatus; import org.apache.log4j.Logger; -import org.fogbowcloud.saps.engine.scheduler.util.ProcessedImagesEmailBuilder; +import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.fogbowcloud.saps.engine.core.model.ImageTaskState; +import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFileList; +import org.fogbowcloud.saps.engine.core.service.EmailService; +import org.fogbowcloud.saps.engine.core.service.ProcessedImagesService; +import org.json.JSONArray; +import org.json.JSONException; import org.restlet.data.Form; +import org.restlet.data.Header; import org.restlet.data.MediaType; +import org.restlet.ext.json.JsonConverter; import org.restlet.representation.Representation; import org.restlet.representation.StringRepresentation; +import org.restlet.resource.ClientResource; import org.restlet.resource.Post; import org.restlet.resource.ResourceException; +import org.restlet.util.Series; -import java.util.Arrays; -import java.util.Properties; +import java.io.IOException; +import java.sql.SQLException; +import java.util.*; +import java.util.stream.Collectors; +/** + * Resource to handle email sending about ImageTasks. + */ public class ImagesEmailResource extends BaseResource { public static final Logger LOGGER = Logger.getLogger(ImagesEmailResource.class); + private static final int JSONARRAY_TO_STRING_FACTOR = 2; private static final String REQUEST_ATTR_PROCESSED_IMAGES = "images_id[]"; + private static final String PROCESSED_TASKS_URN = "/archivedTask"; + private static final String PROCESSED_IMAGES_EMAIL_TITLE = "[SAPS] Filter results"; + private static final String SUCCESSFUL_EMAIL_SENDING_FEEDBACK = "Email será enviado em breve."; + private static final String UNSUCCESSFUL_EMAIL_SENDING_FEEDBACK = "Houve um erro ao enviar o email, tente novamente."; + public static final String RESTLET_HTTP_HEADERS = "org.restlet.http.headers"; + /** + * Sends information of files generated by execution of ImageTasks that + * have had its IDs specified via HTTP body. + * + * @param representation Representation containing data sent via HTTP body. + * @return Representation with feedback of email sending. + */ @Post public Representation sendProcessedImagesToEmail(Representation representation) { Form form = new Form(representation); - String userEmail = form.getFirstValue(UserResource.REQUEST_ATTR_USER_EMAIL, true); String userPass = form.getFirstValue(UserResource.REQUEST_ATTR_USERPASS, true); if (!authenticateUser(userEmail, userPass) || userEmail.equals("anonymous")) { throw new ResourceException(HttpStatus.SC_UNAUTHORIZED); } - - String[] imageIds = form.getValuesArray(REQUEST_ATTR_PROCESSED_IMAGES, true); - Properties properties = application.getProperties(); - ProcessedImagesEmailBuilder emailBuilder = new ProcessedImagesEmailBuilder( - application, - properties, - userEmail, - Arrays.asList(imageIds) - ); + String[] imageTasksIds = form.getValuesArray(REQUEST_ATTR_PROCESSED_IMAGES, true); + try { + List allImageTasksFileList = getImageTaskFileLists(imageTasksIds); + String emailMessage = generateEmailMessage(allImageTasksFileList); + Properties properties = application.getProperties(); + EmailService.sendEmail( + properties, + userEmail, + PROCESSED_IMAGES_EMAIL_TITLE, + emailMessage); + return new StringRepresentation(SUCCESSFUL_EMAIL_SENDING_FEEDBACK, MediaType.TEXT_PLAIN); + } catch (Throwable t) { + LOGGER.error("Error while sending email of processed images", t); + } + return new StringRepresentation(UNSUCCESSFUL_EMAIL_SENDING_FEEDBACK, MediaType.TEXT_PLAIN); + } + + /** + * Generates a email message from list of {@link ImageTaskFileList}. + * + * @param imageTasksFileLists List of ImageTaskFileList. + * @return Email message. + */ + private String generateEmailMessage(List imageTasksFileLists) throws JSONException { + JSONArray emailMessageJSON = generateEmailMessageJSON(imageTasksFileLists); + try { + return emailMessageJSON.toString(JSONARRAY_TO_STRING_FACTOR); + } catch (JSONException e) { + LOGGER.error("Error while generating String email message from JSON", e); + throw e; + } + } + + /** + * Generates a email message in form of JSONArray from list of + * {@link ImageTaskFileList}. + * + * @param imageTasksFileLists List of ImageTaskFileList. + * @return Email message in form of a JSONArray. + */ + private JSONArray generateEmailMessageJSON(List imageTasksFileLists) { + JSONArray emailMessageJSON = new JSONArray(); + try { + for (ImageTaskFileList imageTasksFileList: imageTasksFileLists) { + emailMessageJSON.put(imageTasksFileList.toJSON()); + } + } catch (JSONException e) { + LOGGER.error("Error while parsing list of files to JSON of ImageTask", e); + } + return emailMessageJSON; + } - Thread thread = new Thread(emailBuilder); - thread.start(); + /** + * Returns a list of {@link ImageTaskFileList}, each one of these for each + * {@link ImageTask} that had its ID in specified list of IDs. + * + * @param imageTasksIds List of IDs of ImageTasks. + * @return List of ImageTaskFileList. + */ + private List getImageTaskFileLists(String[] imageTasksIds) throws SQLException { + try { + List imageTasks = application.getImageTasks(imageTasksIds); + List localImageTasksFileLists = getLocalImageTaskFileLists(imageTasks); + List remoteImageTasksFileList = getRemoteImageTaskFileLists(imageTasks); + List allImageTasksFileList = new ArrayList<>(); + allImageTasksFileList.addAll(localImageTasksFileLists); + allImageTasksFileList.addAll(remoteImageTasksFileList); + return allImageTasksFileList; + } catch (SQLException e) { + LOGGER.error("Error while getting list of ImageTaskFileList to send email", e); + throw e; + } + } - return new StringRepresentation("Email será enviado em breve.", MediaType.TEXT_PLAIN); + /** + * Returns a list of {@link ImageTaskFileList}, each one of these for each + * local {@link ImageTask} from specified list of ImageTasks. + * + * @param imageTasks List of ImageTasks containing local ImageTasks. + * @return List of ImageTaskFileList. + */ + private List getLocalImageTaskFileLists(List imageTasks) { + List localImageTasks = imageTasks.stream() + .filter(imageTask -> imageTask.getState().equals(ImageTaskState.ARCHIVED)) + .collect(Collectors.toList()); + return localImageTasks.stream() + .map(imageTask -> ProcessedImagesService.generateImageTaskFiles( + application.getProperties(), + imageTask)) + .collect(Collectors.toList()); } + + /** + * Returns a list of {@link ImageTaskFileList}, each one of these for each + * remote {@link ImageTask} from specified list of ImageTasks. + * + * @param imageTasks List of ImageTasks containing remote ImageTasks. + * @return List of ImageTaskFileList. + */ + private List getRemoteImageTaskFileLists(List imageTasks) { + List remoteImageTasks = imageTasks + .stream() + .filter(imageTask -> imageTask.getState().equals(ImageTaskState.REMOTELY_ARCHIVED)) + .collect(Collectors.toList()); + Map> remoteImageTaskGroupedBySAPSInstanceURL = remoteImageTasks + .stream() + .collect(Collectors.groupingBy(ImageTask::getFederationMember)); + return remoteImageTaskGroupedBySAPSInstanceURL + .entrySet() + .stream() + .map(this::getImageTasksFilesFromSAPSNeighbor) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + } + + /** + * Returns a list of {@link ImageTaskFileList} for SAPS neighbor that had + * it URL specified. + * + * @param SAPSNeighborAndImageTasks {@link Map.Entry} that have a SAPS + * neighbor URL as key and a list of + * ImageTasks as value. + * @return List of {@link ImageTaskFileList}. + */ + private List getImageTasksFilesFromSAPSNeighbor( + Map.Entry> SAPSNeighborAndImageTasks) { + String SAPSNeighborURL = SAPSNeighborAndImageTasks.getKey(); + List imageTasks = SAPSNeighborAndImageTasks.getValue(); + List imageTasksIds = imageTasks.stream() + .map(ImageTask::getTaskId) + .collect(Collectors.toList()); + List imageTaskFileLists = new ArrayList<>(); + try { + ClientResource clientResource = new ClientResource(SAPSNeighborURL + PROCESSED_TASKS_URN); + addImageTasksIdsToHeader(clientResource, imageTasksIds); + Representation response = clientResource.get(MediaType.APPLICATION_JSON); + imageTaskFileLists = extractImageTaskFileLists(response); + } catch (Throwable t) { + LOGGER.error("Error while getting ImageTaskFileList from SAPS Neighbor.", t); + } + return imageTaskFileLists; + } + + /** + * Adds the specified list of IDs of ImageTasks to header of specified + * {@link ClientResource}. + * + * @param clientResource Client Resource to have a header added. + * @param imageTasksIds List of IDs of ImageTasks to be added to header. + */ + @SuppressWarnings("unchecked") + private void addImageTasksIdsToHeader(ClientResource clientResource, List imageTasksIds) { + Series
headers = (Series
) clientResource.getRequestAttributes() + .get(RESTLET_HTTP_HEADERS); + if (headers == null) { + headers = new Series<>(Header.class); + } + // TODO: Verify if sending the toString of array will work + headers.add(REQUEST_ATTR_PROCESSED_IMAGES, imageTasksIds.toString()); + } + + /** + * Extract a list of {@link ImageTaskFileList} from specified response object. + * + * @param response Response containing a list of ImageTaskFileList. + * @return List of ImageTaskFileList. + */ + private List extractImageTaskFileLists(Representation response) + throws IOException, JSONException { + JsonConverter jsonConverter = new JsonConverter(); + JSONArray responseJson = jsonConverter.toObject(response, JSONArray.class, null); + List imageTaskFileLists = new ArrayList<>(); + for (int i = 0; i < responseJson.length(); i++) { + imageTaskFileLists.add(new ImageTaskFileList(responseJson.optJSONObject(i))); + } + return imageTaskFileLists; + } + } diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java deleted file mode 100644 index 75f3a8c32..000000000 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java +++ /dev/null @@ -1,145 +0,0 @@ -package org.fogbowcloud.saps.engine.scheduler.util; - -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.log4j.Logger; -import org.fogbowcloud.saps.engine.core.model.ImageTask; -import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFile; -import org.fogbowcloud.saps.engine.core.service.ProcessedImagesService; -import org.fogbowcloud.saps.engine.scheduler.restlet.DatabaseApplication; -import org.fogbowcloud.saps.notifier.GoogleMail; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -import javax.mail.MessagingException; -import java.sql.SQLException; -import java.util.List; -import java.util.Properties; - -public class ProcessedImagesEmailBuilder implements Runnable { - - private static final Logger LOGGER = Logger.getLogger(ProcessedImagesEmailBuilder.class); - - private static final String UNAVAILABLE = "UNAVAILABLE"; - private static final String TASK_ID = "taskId"; - private static final String REGION = "region"; - private static final String COLLECTION_TIER_NAME = "collectionTierName"; - private static final String IMAGE_DATE = "imageDate"; - private static final String NAME = "name"; - private static final String URL = "url"; - private static final String FILES = "files"; - private static final String STATUS = "status"; - - private DatabaseApplication application; - private Properties properties; - private String userEmail; - private List imageTasksIds; - private StringBuilder error; - - public ProcessedImagesEmailBuilder( - DatabaseApplication databaseApplication, - Properties properties, - String userEmail, - List imageTasksIds) { - this.application = databaseApplication; - this.properties = properties; - this.userEmail = userEmail; - this.imageTasksIds = imageTasksIds; - this.error = new StringBuilder(); - } - - @Override - public void run() { - StringBuilder builder = new StringBuilder(); - builder.append("Creating email for user "); - builder.append(userEmail); - builder.append(" with images:\n"); - for (String imageTaskId: imageTasksIds) { - builder.append(imageTaskId).append("\n"); - } - LOGGER.info(builder.toString()); - JSONArray tasklist = generateAllTasksJsons(); - sendTaskEmail(tasklist); - sendErrorEmail(); - } - - JSONArray generateAllTasksJsons() { - JSONArray tasklist = new JSONArray(); - for (String imageTaskId: imageTasksIds) { - try { - tasklist.put(generateTaskEmailJson(properties, imageTaskId)); - } catch (SQLException e) { - LOGGER.error("Failed to fetch image from database.", e); - error.append("Failed to fetch image from database.").append("\n") - .append(ExceptionUtils.getStackTrace(e)).append("\n"); - } catch (JSONException e) { - LOGGER.error("Failed to create task json.", e); - error.append("Failed to create task json.").append("\n") - .append(ExceptionUtils.getStackTrace(e)).append("\n"); - } - } - return tasklist; - } - - private void sendTaskEmail(JSONArray tasklist) { - try { - GoogleMail.Send( - properties.getProperty(SapsPropertiesConstants.NO_REPLY_EMAIL), - properties.getProperty(SapsPropertiesConstants.NO_REPLY_PASS), - userEmail, - "[SAPS] Filter results", - tasklist.toString(2) - ); - } catch (MessagingException | JSONException e) { - LOGGER.error("Failed to send email with images download links.", e); - error - .append("Failed to send email with images download links.").append("\n") - .append(ExceptionUtils.getStackTrace(e)).append("\n"); - } - } - - private void sendErrorEmail() { - if (!error.toString().isEmpty()) { - try { - GoogleMail.Send( - properties.getProperty(SapsPropertiesConstants.NO_REPLY_EMAIL), - properties.getProperty(SapsPropertiesConstants.NO_REPLY_PASS), - "sebal.no.reply@gmail.com", - "[SAPS] Errors during image temporary link creation", - error.toString() - ); - } catch (MessagingException e) { - LOGGER.error("Failed to send email with errors to admins.", e); - } - } - } - - JSONObject generateTaskEmailJson(Properties properties, String imageTaskId) - throws SQLException, JSONException { - JSONObject result = new JSONObject(); - result.put(TASK_ID, imageTaskId); - - try { - ImageTask task = application.getTask(imageTaskId); - result.put(REGION, task.getRegion()); - result.put(COLLECTION_TIER_NAME, task.getCollectionTierName()); - result.put(IMAGE_DATE, task.getImageDate()); - List imageTaskFiles = ProcessedImagesService - .generateImageTaskFiles(properties, imageTaskId); - JSONArray filesJSONArray = new JSONArray(); - for (ImageTaskFile imageTaskFile: imageTaskFiles) { - JSONObject imageTaskFileJSONObject = new JSONObject(); - imageTaskFileJSONObject.put(NAME, imageTaskFile.getName()); - imageTaskFileJSONObject.put(URL, imageTaskFile.getURL()); - filesJSONArray.put(imageTaskFileJSONObject); - } - result.put(FILES, filesJSONArray); - } catch (SQLException | JSONException e) { - result.put(STATUS, UNAVAILABLE); - throw e; - } - - return result; - } - -} diff --git a/src/test/java/org/fogbowcloud/saps/engine/scheduler/util/TestProcessedImagesEmailBuilder.java b/src/test/java/org/fogbowcloud/saps/engine/scheduler/util/TestProcessedImagesEmailBuilder.java deleted file mode 100644 index 9b0b08aeb..000000000 --- a/src/test/java/org/fogbowcloud/saps/engine/scheduler/util/TestProcessedImagesEmailBuilder.java +++ /dev/null @@ -1,280 +0,0 @@ -package org.fogbowcloud.saps.engine.scheduler.util; - -import org.fogbowcloud.saps.engine.core.model.ImageTask; -import org.fogbowcloud.saps.engine.core.model.ImageTaskState; -import org.fogbowcloud.saps.engine.scheduler.restlet.DatabaseApplication; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Matchers; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; -import java.security.SignatureException; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Properties; - -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; - -public class TestProcessedImagesEmailBuilder { - - private static final String IMAGE_TASK_ID = "task_id_101010101010"; - private static final String UNAVAILABLE = "UNAVAILABLE"; - - @Test - public void testGenerateTaskEmailJson() throws SQLException, IOException, URISyntaxException, JSONException { - Properties properties = getProperties(); - ImageTask imageTask = getImageTask(); - List imageIds = getImageIdList(); - DatabaseApplication application = getDatabaseApplication(imageTask); - ProcessedImagesEmailBuilder emailBuilder = getProcessedImagesEmailBuilder(properties, imageIds, application); - - List imageFiles = getImageFileList(); - doReturn(imageFiles).when(emailBuilder).getTaskFilesFromObjectStore( - any(Properties.class), anyString(), anyString(), anyString(), any(ImageTask.class) - ); - - JSONObject jsonReturn; - jsonReturn = emailBuilder.generateTaskEmailJson(properties, IMAGE_TASK_ID); - - Assert.assertEquals(imageTask.getTaskId(), jsonReturn.getString("taskId")); - Assert.assertEquals(imageTask.getRegion(), jsonReturn.getString("region")); - Assert.assertEquals(imageTask.getImageDate().toString(), jsonReturn.getString("imageDate")); - Assert.assertEquals(imageTask.getCollectionTierName(), jsonReturn.getString("collectionTierName")); - - JSONArray files = jsonReturn.getJSONArray("files"); - Assert.assertEquals(imageFiles.size(), files.length()); - for (int i = 0; i < files.length(); i++) { - boolean missing = true; - for (String imageFile : imageFiles) { - JSONObject f = files.getJSONObject(i); - String name = f.getString("name"); - if (name.equals(imageFile)) { - String url = f.getString("url"); - System.out.println(name + ": " + url); - checkUrlAvailable(properties, name, url); - missing = false; - break; - } - } - if (missing) { - Assert.fail("One of the files is missing"); - } - } - } - - @Test - public void testGenerateTaskEmailJsonErrorGeneratingURL() throws SQLException, IOException, URISyntaxException, NoSuchAlgorithmException, SignatureException, InvalidKeyException, JSONException { - Properties properties = getProperties(); - ImageTask imageTask = getImageTask(); - List imageIds = getImageIdList(); - DatabaseApplication application = getDatabaseApplication(imageTask); - ProcessedImagesEmailBuilder emailBuilder = getProcessedImagesEmailBuilder(properties, imageIds, application); - - List imageFiles = getImageFileList(); - String failingFile = imageFiles.get(3); - - doReturn(imageFiles).when(emailBuilder).getTaskFilesFromObjectStore( - any(Properties.class), anyString(), anyString(), anyString(), any(ImageTask.class) - ); - doThrow(SignatureException.class).when(emailBuilder).generateTempURL( - anyString(), - anyString(), - Matchers.eq(failingFile), - anyString() - ); - - JSONObject jsonReturn; - jsonReturn = emailBuilder.generateTaskEmailJson(properties, IMAGE_TASK_ID); - - Assert.assertEquals(imageTask.getTaskId(), jsonReturn.getString("taskId")); - Assert.assertEquals(imageTask.getRegion(), jsonReturn.getString("region")); - Assert.assertEquals(imageTask.getImageDate().toString(), jsonReturn.getString("imageDate")); - Assert.assertEquals(imageTask.getCollectionTierName(), jsonReturn.getString("collectionTierName")); - - JSONArray files = jsonReturn.getJSONArray("files"); - Assert.assertEquals(imageFiles.size(), files.length()); - for (int i = 0; i < files.length(); i++) { - boolean missing = true; - for (String imageFile : imageFiles) { - JSONObject f = files.getJSONObject(i); - String name = f.getString("name"); - if (name.equals(imageFile)) { - String url = f.getString("url"); - System.out.println(name + ": " + url); - if (name.equals(failingFile)) { - Assert.assertEquals(UNAVAILABLE, url); - } else { - checkUrlAvailable(properties, name, url); - } - missing = false; - break; - } - } - if (missing) { - Assert.fail("One of the files is missing"); - } - } - } - - @Test - public void testGenerateTaskEmailJsonErrorOnDatabase() throws SQLException, IOException, URISyntaxException, NoSuchAlgorithmException, SignatureException, InvalidKeyException, JSONException { - Properties properties = getProperties(); - ImageTask imageTask = getImageTask(); - List imageIds = getImageIdList(); - DatabaseApplication application = getDatabaseApplication(imageTask); - ProcessedImagesEmailBuilder emailBuilder = getProcessedImagesEmailBuilder(properties, imageIds, application); - - try { - emailBuilder.generateTaskEmailJson(properties, "FAIL_TASK"); - Assert.fail("Should throw exception"); - } catch (SQLException e) { - e.printStackTrace(); - } - } - - @Test - public void testGenerateAllTasksJsons() throws SQLException, JSONException, IOException, URISyntaxException { - Properties properties = getProperties(); - ImageTask imageTask = getImageTask(); - List imageIds = getImageIdList(); - imageIds.add("FAIL_TASK"); - DatabaseApplication application = getDatabaseApplication(imageTask); - ProcessedImagesEmailBuilder emailBuilder = getProcessedImagesEmailBuilder(properties, imageIds, application); - - List imageFiles = getImageFileList(); - doReturn(imageFiles).when(emailBuilder).getTaskFilesFromObjectStore( - any(Properties.class), anyString(), anyString(), anyString(), any(ImageTask.class) - ); - - StringBuilder errorBuilder = new StringBuilder(); - JSONArray tasks = emailBuilder.generateAllTasksJsons(errorBuilder); - for (int i = 0; i < tasks.length(); i++) { - JSONObject obj = tasks.getJSONObject(i); - boolean missing = true; - for (String tid: imageIds) { - String name = obj.getString("taskId"); - if (name.equals(tid)) { - System.out.println(obj.toString(2)); - String status = obj.optString("status"); - if (name.equals("FAIL_TASK")) { - Assert.assertEquals(UNAVAILABLE, status); - } else { - Assert.assertNotEquals(UNAVAILABLE, status); - } - missing = false; - break; - } - } - if (missing) { - Assert.fail("One of the tasks is missing"); - } - } - } - - private DatabaseApplication getDatabaseApplication(final ImageTask imageTask) throws SQLException { - DatabaseApplication application = Mockito.mock(DatabaseApplication.class); - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock invocationOnMock) throws Throwable { - Object[] args = invocationOnMock.getArguments(); - String imageId = (String)args[0]; - if (imageId.equals(imageTask.getTaskId())) return imageTask; - else throw new SQLException(); - } - }).when(application).getTask(anyString()); - return application; - } - - private Properties getProperties() { - Properties properties = new Properties(); - properties.setProperty(SapsPropertiesConstants.SWIFT_AUTH_URL, "https://cloud.lsd.ufcg.edu.br:5000"); - properties.setProperty(SapsPropertiesConstants.SWIFT_PROJECT_ID, "3324431f606d4a74a060cf78c16fcb21"); - properties.setProperty(SapsPropertiesConstants.SWIFT_USER_ID, "3e57892203271c195f5d473fc84f484b8062103275ce6ad6e7bcd1baedf70d5c"); - properties.setProperty(SapsPropertiesConstants.SWIFT_PASSWORD, "123456"); - properties.setProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_HOST, "cloud.lsd.ufcg.edu.br:8080"); - properties.setProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_PATH, "/swift/v1"); - properties.setProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_CONTAINER, "lsd_deploy"); - properties.setProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_KEY, "qnjcKtALnbqHkOM0f0J9I8TWzANCI7Qj8"); - properties.setProperty( - SapsPropertiesConstants.NO_REPLY_EMAIL, "sebal.no.reply@gmail.com" - ); - properties.setProperty( - SapsPropertiesConstants.NO_REPLY_PASS, "noREsebal16" - ); - return properties; - } - - private List getImageIdList() { - List imageIds = new ArrayList<>(); - imageIds.add(IMAGE_TASK_ID); - return imageIds; - } - - private ImageTask getImageTask() { - return new ImageTask( - IMAGE_TASK_ID, - "landsat-7", - "263_065", - new Date(), - "NA", - ImageTaskState.ARCHIVED, - "NA", - 0, - "NA", - "Default", - "Default", - "Default", - "NA", - "NA", - new Timestamp(System.nanoTime()), - new Timestamp(System.nanoTime()), - "NA", - "NA" - ); - } - - private ProcessedImagesEmailBuilder getProcessedImagesEmailBuilder(Properties properties, List imageIds, DatabaseApplication application) { - return Mockito.spy( - new ProcessedImagesEmailBuilder( - application, properties, "sebal.no.replay@gmail.com", imageIds - ) - ); - } - - private void checkUrlAvailable(Properties properties, String name, String url) { - Assert.assertNotEquals(UNAVAILABLE, url); - Assert.assertTrue(url.contains(properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_HOST))); - Assert.assertTrue(url.contains(properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_PATH))); - Assert.assertTrue(url.contains(properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_CONTAINER))); - Assert.assertTrue(url.contains(name)); - Assert.assertTrue(url.contains("?temp_url_sig")); - Assert.assertTrue(url.contains("&temp_url_expires")); - Assert.assertTrue(url.contains("&filename="+name)); - } - - private List getImageFileList() { - List imageFiles = new ArrayList<>(); - imageFiles.add("file1"); - imageFiles.add("file2"); - imageFiles.add("file3"); - imageFiles.add("file4"); - return imageFiles; - } - -} From c4d0739b4d6a5dedd0600dddf4cbb5aefd23e271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Fri, 14 Jun 2019 16:05:14 -0300 Subject: [PATCH 14/21] Fix SQL query to find image tasks that have its ID in specified list of IDs --- .../saps/engine/core/database/JDBCImageDataStore.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/database/JDBCImageDataStore.java b/src/main/java/org/fogbowcloud/saps/engine/core/database/JDBCImageDataStore.java index 8bc0e4412..396fa0a81 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/database/JDBCImageDataStore.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/database/JDBCImageDataStore.java @@ -1,6 +1,8 @@ package org.fogbowcloud.saps.engine.core.database; +import java.sql.Array; import java.sql.Connection; +import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -1016,7 +1018,7 @@ public List getAllTasks() throws SQLException { } private static final String SELECT_IMAGES_IN_ID_LIST_SQL = "SELECT * FROM " + IMAGE_TABLE_NAME - + " WHERE " + TASK_ID_COL + " IN ?"; + + " WHERE " + TASK_ID_COL + " IN (?)"; @Override public List getImageTasks(String[] imageTasksIds) throws SQLException { @@ -1025,7 +1027,8 @@ public List getImageTasks(String[] imageTasksIds) throws SQLException try { conn = getConnection(); statement = conn.prepareStatement(SELECT_IMAGES_IN_ID_LIST_SQL); - statement.setString(1, Arrays.toString(imageTasksIds)); + Array imageTasksIdsArray = conn.createArrayOf(JDBCType.VARCHAR.getName(), imageTasksIds); + statement.setArray(1, imageTasksIdsArray); statement.setQueryTimeout(300); statement.execute(); ResultSet rs = statement.getResultSet(); From 55560c1a256bbb0519beb544faf19e052ae0b530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Fri, 14 Jun 2019 20:17:14 -0300 Subject: [PATCH 15/21] Updates way of sending list of IDs of ImageTasks (before it was through header, now it's through query param) --- .../resource/ArchivedTasksResource.java | 13 +++++--- .../restlet/resource/ImagesEmailResource.java | 32 +++++-------------- 2 files changed, 17 insertions(+), 28 deletions(-) diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java index 463604f24..0f53119ea 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import org.apache.log4j.Logger; @@ -31,6 +32,9 @@ public class ArchivedTasksResource extends BaseResource { private static final Logger LOGGER = Logger.getLogger(ArchivedTasksResource.class); private static final String REQUEST_ATTR_PROCESSED_IMAGES = "images_id[]"; + private static final String IMAGETASKS_IDS_SEPARATOR = ","; + private static final String LIST_OF_IDS_OF_IMAGETASKS_REQUIRED = + "A lista de IDs das ImageTasks é obrigatória."; /** * Returns all processed tasks that matches the execution parameters passed @@ -62,13 +66,14 @@ public Representation getProcessedTasksInInterval(Representation representation) * Returns the files generated by execution of each {@link ImageTask} that * had its ID specified. */ - @SuppressWarnings("unchecked") @Get public Representation getProcessedImageTasksFiles() { - Series
series = (Series
) getRequestAttributes() - .get("org.restlet.http.headers"); + String imageTasksIdsParam = getQuery().getValues(REQUEST_ATTR_PROCESSED_IMAGES); + if (Objects.isNull(imageTasksIdsParam)) { + return new StringRepresentation(LIST_OF_IDS_OF_IMAGETASKS_REQUIRED, MediaType.TEXT_PLAIN); + } - String[] imageTasksIds = series.getValuesArray(REQUEST_ATTR_PROCESSED_IMAGES, true); + String[] imageTasksIds = imageTasksIdsParam.split(IMAGETASKS_IDS_SEPARATOR); LOGGER.info("Recovering files from processed ImageTasks from list of IDs: " + Arrays.toString(imageTasksIds)); diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java index 4b61c20f0..86ef4678e 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java @@ -10,7 +10,6 @@ import org.json.JSONArray; import org.json.JSONException; import org.restlet.data.Form; -import org.restlet.data.Header; import org.restlet.data.MediaType; import org.restlet.ext.json.JsonConverter; import org.restlet.representation.Representation; @@ -18,7 +17,6 @@ import org.restlet.resource.ClientResource; import org.restlet.resource.Post; import org.restlet.resource.ResourceException; -import org.restlet.util.Series; import java.io.IOException; import java.sql.SQLException; @@ -36,9 +34,11 @@ public class ImagesEmailResource extends BaseResource { private static final String REQUEST_ATTR_PROCESSED_IMAGES = "images_id[]"; private static final String PROCESSED_TASKS_URN = "/archivedTask"; private static final String PROCESSED_IMAGES_EMAIL_TITLE = "[SAPS] Filter results"; - private static final String SUCCESSFUL_EMAIL_SENDING_FEEDBACK = "Email será enviado em breve."; - private static final String UNSUCCESSFUL_EMAIL_SENDING_FEEDBACK = "Houve um erro ao enviar o email, tente novamente."; - public static final String RESTLET_HTTP_HEADERS = "org.restlet.http.headers"; + private static final String SUCCESSFUL_EMAIL_SENDING_FEEDBACK = + "Email será enviado em breve."; + private static final String UNSUCCESSFUL_EMAIL_SENDING_FEEDBACK = + "Houve um erro ao enviar o email, tente novamente."; + private static final String RESTLET_HTTP_HEADERS = "org.restlet.http.headers"; /** * Sends information of files generated by execution of ImageTasks that @@ -190,7 +190,9 @@ private List getImageTasksFilesFromSAPSNeighbor( List imageTaskFileLists = new ArrayList<>(); try { ClientResource clientResource = new ClientResource(SAPSNeighborURL + PROCESSED_TASKS_URN); - addImageTasksIdsToHeader(clientResource, imageTasksIds); + for (String imageTaskId: imageTasksIds) { + clientResource.addQueryParameter(REQUEST_ATTR_PROCESSED_IMAGES, imageTaskId); + } Representation response = clientResource.get(MediaType.APPLICATION_JSON); imageTaskFileLists = extractImageTaskFileLists(response); } catch (Throwable t) { @@ -199,24 +201,6 @@ private List getImageTasksFilesFromSAPSNeighbor( return imageTaskFileLists; } - /** - * Adds the specified list of IDs of ImageTasks to header of specified - * {@link ClientResource}. - * - * @param clientResource Client Resource to have a header added. - * @param imageTasksIds List of IDs of ImageTasks to be added to header. - */ - @SuppressWarnings("unchecked") - private void addImageTasksIdsToHeader(ClientResource clientResource, List imageTasksIds) { - Series
headers = (Series
) clientResource.getRequestAttributes() - .get(RESTLET_HTTP_HEADERS); - if (headers == null) { - headers = new Series<>(Header.class); - } - // TODO: Verify if sending the toString of array will work - headers.add(REQUEST_ATTR_PROCESSED_IMAGES, imageTasksIds.toString()); - } - /** * Extract a list of {@link ImageTaskFileList} from specified response object. * From c181d4e42ad4f38c47cfaca21ce1c81fd05f11b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Fri, 14 Jun 2019 21:55:32 -0300 Subject: [PATCH 16/21] Adds missing setter methods --- .../fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java index 6c1ecb47c..f6d2c0e43 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java @@ -25,10 +25,18 @@ public String getPath() { return path; } + public void setPath(String path) { + this.path = path; + } + public String getName() { return name; } + public void setName(String name) { + this.name = name; + } + public String getURL() { return URL; } @@ -36,4 +44,5 @@ public String getURL() { public void setURL(String URL) { this.URL = URL; } + } From 3625889ed178632b3be5f919f9c7b478ab4c5945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Fri, 14 Jun 2019 22:01:01 -0300 Subject: [PATCH 17/21] Adds error message when getting of files of ImageTask fails --- .../engine/core/service/ObjectStoreService.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/service/ObjectStoreService.java b/src/main/java/org/fogbowcloud/saps/engine/core/service/ObjectStoreService.java index a90e653c3..d73fa63f0 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/service/ObjectStoreService.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/service/ObjectStoreService.java @@ -7,6 +7,8 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import org.apache.log4j.Logger; +import org.fogbowcloud.manager.core.plugins.identity.openstack.KeystoneV3IdentityPlugin; +import org.fogbowcloud.manager.occi.model.Token; import org.fogbowcloud.saps.engine.core.model.ImageTask; import org.fogbowcloud.saps.engine.scheduler.util.SapsPropertiesConstants; @@ -16,7 +18,7 @@ import java.util.*; /** - * Service that provides operations for communication with the Object Store. + * Service that provides operations for communication with Object Store. */ public class ObjectStoreService { @@ -42,16 +44,17 @@ public class ObjectStoreService { * @return List of paths. */ public static List getImageTaskFilesPaths(Properties properties, String imageTaskId) { + List imageTaskFilesPaths = new ArrayList<>(); try { HttpClient client = HttpClients.createDefault(); HttpGet httpget = prepareObjectStoreRequest(properties, imageTaskId); HttpResponse response = client.execute(httpget); - return parseHttpResponse(response); + imageTaskFilesPaths = parseHttpResponse(response); } catch (IOException | URISyntaxException e) { - LOGGER.error("", e); - return Collections.emptyList(); + LOGGER.error("Error while retrieving path of files of ImageTask" + + " with ID: " + imageTaskId, e); } - + return imageTaskFilesPaths; } /** @@ -107,4 +110,5 @@ private static Token getKeystoneToken(Properties properties) { KeystoneV3IdentityPlugin keystone = new KeystoneV3IdentityPlugin(properties); return keystone.createToken(credentials); } + } From e34b94f34a79cf5a227a5d95e42c3253f076e53f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Fri, 14 Jun 2019 22:05:02 -0300 Subject: [PATCH 18/21] Undo unnecessary modifications in `ProcessedImagesEmailBuilder` --- .../util/ProcessedImagesEmailBuilder.java | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java index 75f3a8c32..a16a8b256 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java @@ -33,8 +33,7 @@ public class ProcessedImagesEmailBuilder implements Runnable { private DatabaseApplication application; private Properties properties; private String userEmail; - private List imageTasksIds; - private StringBuilder error; + private List images; public ProcessedImagesEmailBuilder( DatabaseApplication databaseApplication, @@ -44,8 +43,7 @@ public ProcessedImagesEmailBuilder( this.application = databaseApplication; this.properties = properties; this.userEmail = userEmail; - this.imageTasksIds = imageTasksIds; - this.error = new StringBuilder(); + this.images = images; } @Override @@ -54,34 +52,35 @@ public void run() { builder.append("Creating email for user "); builder.append(userEmail); builder.append(" with images:\n"); - for (String imageTaskId: imageTasksIds) { - builder.append(imageTaskId).append("\n"); + for (String str: images) { + builder.append(str).append("\n"); } LOGGER.info(builder.toString()); - JSONArray tasklist = generateAllTasksJsons(); + StringBuilder errorBuilder = new StringBuilder(); + JSONArray tasklist = generateAllTasksJsons(errorBuilder); sendTaskEmail(tasklist); sendErrorEmail(); } - JSONArray generateAllTasksJsons() { + JSONArray generateAllTasksJsons(StringBuilder errorBuilder) { JSONArray tasklist = new JSONArray(); - for (String imageTaskId: imageTasksIds) { + for (String str: images) { try { - tasklist.put(generateTaskEmailJson(properties, imageTaskId)); + tasklist.put(generateTaskEmailJson(properties, str)); } catch (SQLException e) { LOGGER.error("Failed to fetch image from database.", e); - error.append("Failed to fetch image from database.").append("\n") + errorBuilder.append("Failed to fetch image from database.").append("\n") .append(ExceptionUtils.getStackTrace(e)).append("\n"); } catch (JSONException e) { LOGGER.error("Failed to create task json.", e); - error.append("Failed to create task json.").append("\n") + errorBuilder.append("Failed to create task json.").append("\n") .append(ExceptionUtils.getStackTrace(e)).append("\n"); } } return tasklist; } - private void sendTaskEmail(JSONArray tasklist) { + private void sendTaskEmail(StringBuilder errorBuilder, JSONArray tasklist) { try { GoogleMail.Send( properties.getProperty(SapsPropertiesConstants.NO_REPLY_EMAIL), @@ -92,21 +91,21 @@ private void sendTaskEmail(JSONArray tasklist) { ); } catch (MessagingException | JSONException e) { LOGGER.error("Failed to send email with images download links.", e); - error + errorBuilder .append("Failed to send email with images download links.").append("\n") .append(ExceptionUtils.getStackTrace(e)).append("\n"); } } - private void sendErrorEmail() { - if (!error.toString().isEmpty()) { + private void sendErrorEmail(StringBuilder errorBuilder) { + if (!errorBuilder.toString().isEmpty()) { try { GoogleMail.Send( properties.getProperty(SapsPropertiesConstants.NO_REPLY_EMAIL), properties.getProperty(SapsPropertiesConstants.NO_REPLY_PASS), "sebal.no.reply@gmail.com", "[SAPS] Errors during image temporary link creation", - error.toString() + errorBuilder.toString() ); } catch (MessagingException e) { LOGGER.error("Failed to send email with errors to admins.", e); From 55612e0fbbc657b2714681803211a652bf9166d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Fri, 14 Jun 2019 22:20:05 -0300 Subject: [PATCH 19/21] Utilizes extracted `ProcessedImagesService` in `ProcessedImagesEmailBuilder` --- .../util/ProcessedImagesEmailBuilder.java | 46 ++++--------------- 1 file changed, 8 insertions(+), 38 deletions(-) diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java index a16a8b256..480f08f06 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java @@ -3,7 +3,7 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.log4j.Logger; import org.fogbowcloud.saps.engine.core.model.ImageTask; -import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFile; +import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFileList; import org.fogbowcloud.saps.engine.core.service.ProcessedImagesService; import org.fogbowcloud.saps.engine.scheduler.restlet.DatabaseApplication; import org.fogbowcloud.saps.notifier.GoogleMail; @@ -20,16 +20,6 @@ public class ProcessedImagesEmailBuilder implements Runnable { private static final Logger LOGGER = Logger.getLogger(ProcessedImagesEmailBuilder.class); - private static final String UNAVAILABLE = "UNAVAILABLE"; - private static final String TASK_ID = "taskId"; - private static final String REGION = "region"; - private static final String COLLECTION_TIER_NAME = "collectionTierName"; - private static final String IMAGE_DATE = "imageDate"; - private static final String NAME = "name"; - private static final String URL = "url"; - private static final String FILES = "files"; - private static final String STATUS = "status"; - private DatabaseApplication application; private Properties properties; private String userEmail; @@ -39,7 +29,7 @@ public ProcessedImagesEmailBuilder( DatabaseApplication databaseApplication, Properties properties, String userEmail, - List imageTasksIds) { + List images) { this.application = databaseApplication; this.properties = properties; this.userEmail = userEmail; @@ -58,8 +48,8 @@ public void run() { LOGGER.info(builder.toString()); StringBuilder errorBuilder = new StringBuilder(); JSONArray tasklist = generateAllTasksJsons(errorBuilder); - sendTaskEmail(tasklist); - sendErrorEmail(); + sendTaskEmail(errorBuilder, tasklist); + sendErrorEmail(errorBuilder); } JSONArray generateAllTasksJsons(StringBuilder errorBuilder) { @@ -115,30 +105,10 @@ private void sendErrorEmail(StringBuilder errorBuilder) { JSONObject generateTaskEmailJson(Properties properties, String imageTaskId) throws SQLException, JSONException { - JSONObject result = new JSONObject(); - result.put(TASK_ID, imageTaskId); - - try { - ImageTask task = application.getTask(imageTaskId); - result.put(REGION, task.getRegion()); - result.put(COLLECTION_TIER_NAME, task.getCollectionTierName()); - result.put(IMAGE_DATE, task.getImageDate()); - List imageTaskFiles = ProcessedImagesService - .generateImageTaskFiles(properties, imageTaskId); - JSONArray filesJSONArray = new JSONArray(); - for (ImageTaskFile imageTaskFile: imageTaskFiles) { - JSONObject imageTaskFileJSONObject = new JSONObject(); - imageTaskFileJSONObject.put(NAME, imageTaskFile.getName()); - imageTaskFileJSONObject.put(URL, imageTaskFile.getURL()); - filesJSONArray.put(imageTaskFileJSONObject); - } - result.put(FILES, filesJSONArray); - } catch (SQLException | JSONException e) { - result.put(STATUS, UNAVAILABLE); - throw e; - } - - return result; + ImageTask task = application.getTask(imageTaskId); + ImageTaskFileList imageTaskFileList = ProcessedImagesService + .generateImageTaskFiles(properties, task); + return imageTaskFileList.toJSON(); } } From 730acf172704e22474f14ef8e290059c85e60396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Mon, 17 Jun 2019 11:08:48 -0300 Subject: [PATCH 20/21] Fixes javadoc and constant name --- .../scheduler/restlet/resource/ImagesEmailResource.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java index d6a36c958..8019212d5 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java @@ -30,9 +30,9 @@ public class ImagesEmailResource extends BaseResource { public static final Logger LOGGER = Logger.getLogger(ImagesEmailResource.class); - private static final int JSONARRAY_TO_STRING_FACTOR = 2; + private static final int JSON_ARRAY_TO_STRING_FACTOR = 2; private static final String REQUEST_ATTR_PROCESSED_IMAGES = "images_id[]"; - private static final String PROCESSED_TASKS_URN = "/archivedTask"; + private static final String PROCESSED_TASKS_URN = "/archivedTasks"; private static final String PROCESSED_IMAGES_EMAIL_TITLE = "[SAPS] Filter results"; private static final String SUCCESSFUL_EMAIL_SENDING_FEEDBACK = "Email será enviado em breve."; @@ -81,7 +81,7 @@ public Representation sendProcessedImagesToEmail(Representation representation) private String generateEmailMessage(List imageTasksFileLists) throws JSONException { JSONArray emailMessageJSON = generateEmailMessageJSON(imageTasksFileLists); try { - return emailMessageJSON.toString(JSONARRAY_TO_STRING_FACTOR); + return emailMessageJSON.toString(JSON_ARRAY_TO_STRING_FACTOR); } catch (JSONException e) { LOGGER.error("Error while generating String email message from JSON", e); throw e; @@ -172,7 +172,7 @@ private List getRemoteImageTaskFileLists(List imag /** * Returns a list of {@link ImageTaskFileList} for SAPS neighbor that had - * it URL specified. + * its URL specified. * * @param SAPSNeighborAndImageTasks {@link Map.Entry} that have a SAPS * neighbor URL as key and a list of From 77f296558342795f29acb2cf22f4053761a17ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A9lmer=20Oliveira?= Date: Mon, 17 Jun 2019 11:20:28 -0300 Subject: [PATCH 21/21] Creates toJSON and constructor from JSON for `ImageTaskFile` --- .../saps/engine/core/pojo/ImageTaskFile.java | 18 ++++++++++++++++++ .../engine/core/pojo/ImageTaskFileList.java | 18 ++++-------------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java index f6d2c0e43..7ab2ddf9a 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java @@ -1,12 +1,17 @@ package org.fogbowcloud.saps.engine.core.pojo; import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.json.JSONException; +import org.json.JSONObject; /** * Holds the information of a file generated by the processing of a {@link ImageTask}. */ public class ImageTaskFile { + private static final String NAME_KEY = "name"; + private static final String URL_KEY = "url"; + private String path; private String name; private String URL; @@ -21,6 +26,19 @@ public ImageTaskFile(String path, String name, String URL) { this.URL = URL; } + public ImageTaskFile(JSONObject jsonObject) throws JSONException { + this(null, + jsonObject.getString(NAME_KEY), + jsonObject.getString(URL_KEY)); + } + + public JSONObject toJSON() throws JSONException { + JSONObject imageTaskFileJSONObject = new JSONObject(); + imageTaskFileJSONObject.put(NAME_KEY, name); + imageTaskFileJSONObject.put(URL_KEY, URL); + return imageTaskFileJSONObject; + } + public String getPath() { return path; } diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java index 2332a9278..359c1069e 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java @@ -18,8 +18,6 @@ public class ImageTaskFileList { private static final String REGION = "region"; private static final String COLLECTION_TIER_NAME = "collectionTierName"; private static final String IMAGE_DATE = "imageDate"; - private static final String NAME = "name"; - private static final String URL = "url"; private static final String FILES = "files"; private static final String STATUS = "status"; private static final String UNAVAILABLE = "UNAVAILABLE"; @@ -40,12 +38,7 @@ public ImageTaskFileList(JSONObject jsonObject) throws JSONException { List imageTaskFiles = new ArrayList<>(); JSONArray filesJSONArray = jsonObject.getJSONArray(FILES); for (int i = 0; i < filesJSONArray.length(); i++) { - JSONObject imageTaskFileJSON = filesJSONArray.optJSONObject(i); - imageTaskFiles.add(new ImageTaskFile( - null, - imageTaskFileJSON.getString(NAME), - imageTaskFileJSON.getString(URL) - )); + imageTaskFiles.add(new ImageTaskFile(filesJSONArray.optJSONObject(i))); } this.imageTask = imageTask; this.imageTaskFiles = imageTaskFiles; @@ -58,14 +51,11 @@ public JSONObject toJSON() throws JSONException { jsonObject.put(REGION, imageTask.getRegion()); jsonObject.put(IMAGE_DATE, imageTask.getImageDate()); jsonObject.put(COLLECTION_TIER_NAME, imageTask.getCollectionTierName()); - JSONArray filesJSONArray = new JSONArray(); + JSONArray imageTaskFilesJSONArray = new JSONArray(); for (ImageTaskFile imageTaskFile : imageTaskFiles) { - JSONObject imageTaskFileJSONObject = new JSONObject(); - imageTaskFileJSONObject.put(NAME, imageTaskFile.getName()); - imageTaskFileJSONObject.put(URL, imageTaskFile.getURL()); - filesJSONArray.put(imageTaskFileJSONObject); + imageTaskFilesJSONArray.put(imageTaskFile.toJSON()); } - jsonObject.put(FILES, filesJSONArray); + jsonObject.put(FILES, imageTaskFilesJSONArray); } catch (JSONException e) { jsonObject.put(STATUS, UNAVAILABLE); throw e;