diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/database/ImageDataStore.java b/src/main/java/org/fogbowcloud/saps/engine/core/database/ImageDataStore.java index bf3544325..af57b01a2 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/database/ImageDataStore.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/database/ImageDataStore.java @@ -126,4 +126,6 @@ public List getProcessedImages( String inputGathering, String inputPreprocessing, String algorithmExecution) throws SQLException; + + List getImageTasks(String[] imageTasksIds) throws SQLException; } diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/database/JDBCImageDataStore.java b/src/main/java/org/fogbowcloud/saps/engine/core/database/JDBCImageDataStore.java index aff7d1f4f..396fa0a81 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/database/JDBCImageDataStore.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/database/JDBCImageDataStore.java @@ -1,17 +1,14 @@ package org.fogbowcloud.saps.engine.core.database; +import java.sql.Array; import java.sql.Connection; +import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.dbcp2.BasicDataSource; @@ -1020,6 +1017,27 @@ public List getAllTasks() throws SQLException { } } + private static final String SELECT_IMAGES_IN_ID_LIST_SQL = "SELECT * FROM " + IMAGE_TABLE_NAME + + " WHERE " + TASK_ID_COL + " IN (?)"; + + @Override + public List getImageTasks(String[] imageTasksIds) throws SQLException { + PreparedStatement statement = null; + Connection conn = null; + try { + conn = getConnection(); + statement = conn.prepareStatement(SELECT_IMAGES_IN_ID_LIST_SQL); + Array imageTasksIdsArray = conn.createArrayOf(JDBCType.VARCHAR.getName(), imageTasksIds); + statement.setArray(1, imageTasksIdsArray); + statement.setQueryTimeout(300); + statement.execute(); + ResultSet rs = statement.getResultSet(); + return extractImageTaskFrom(rs); + } finally { + close(statement, conn); + } + } + private static final String SELECT_USER_SQL = "SELECT * FROM " + USERS_TABLE_NAME + " WHERE " + USER_EMAIL_COL + " = ?"; diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java index 4e18161cf..f90771223 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java @@ -41,4 +41,6 @@ void removeUserNotification(String submissionId, String taskId, String userEmail boolean isUserNotifiable(String userEmail) throws SQLException; List searchProcessedTasks(SubmissionParameters submissionParameters); + + List getImageTasks(String[] imageTasksIds) throws SQLException; } diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherImpl.java b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherImpl.java index a189c57f8..5f59886b6 100755 --- a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherImpl.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherImpl.java @@ -269,6 +269,11 @@ public List getTaskListInDB() throws SQLException, ParseException { return imageStore.getAllTasks(); } + @Override + public List getImageTasks(String[] imageTasksIds) throws SQLException { + return imageStore.getImageTasks(imageTasksIds); + } + @Override public List getUsersToNotify() throws SQLException { List wards = imageStore.getUsersToNotify(); diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/model/ImageTask.java b/src/main/java/org/fogbowcloud/saps/engine/core/model/ImageTask.java index 22f13675a..d4a0739b4 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/model/ImageTask.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/model/ImageTask.java @@ -89,7 +89,13 @@ public ImageTask(JSONObject imageTaskJsonObject) throws JSONException { ); } - public String getTaskId() { + public ImageTask(String taskId, String region, Date imageDate) { + this.taskId = taskId; + this.region = region; + this.imageDate = imageDate; + } + + public String getTaskId() { return taskId; } diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java new file mode 100644 index 000000000..7ab2ddf9a --- /dev/null +++ b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java @@ -0,0 +1,66 @@ +package org.fogbowcloud.saps.engine.core.pojo; + +import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.json.JSONException; +import org.json.JSONObject; + +/** + * Holds the information of a file generated by the processing of a {@link ImageTask}. + */ +public class ImageTaskFile { + + private static final String NAME_KEY = "name"; + private static final String URL_KEY = "url"; + + private String path; + private String name; + private String URL; + + public ImageTaskFile(String path, String name) { + this(path, name, null); + } + + public ImageTaskFile(String path, String name, String URL) { + this.path = path; + this.name = name; + this.URL = URL; + } + + public ImageTaskFile(JSONObject jsonObject) throws JSONException { + this(null, + jsonObject.getString(NAME_KEY), + jsonObject.getString(URL_KEY)); + } + + public JSONObject toJSON() throws JSONException { + JSONObject imageTaskFileJSONObject = new JSONObject(); + imageTaskFileJSONObject.put(NAME_KEY, name); + imageTaskFileJSONObject.put(URL_KEY, URL); + return imageTaskFileJSONObject; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getURL() { + return URL; + } + + public void setURL(String URL) { + this.URL = URL; + } + +} diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java new file mode 100644 index 000000000..359c1069e --- /dev/null +++ b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java @@ -0,0 +1,65 @@ +package org.fogbowcloud.saps.engine.core.pojo; + +import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/** + * Represents a list of {@link ImageTaskFile} for a specific {@link ImageTask}. + */ +public class ImageTaskFileList { + + private static final String IMAGE_TASK_ID = "imageTaskId"; + private static final String REGION = "region"; + private static final String COLLECTION_TIER_NAME = "collectionTierName"; + private static final String IMAGE_DATE = "imageDate"; + private static final String FILES = "files"; + private static final String STATUS = "status"; + private static final String UNAVAILABLE = "UNAVAILABLE"; + + private ImageTask imageTask; + private List imageTaskFiles; + + public ImageTaskFileList(ImageTask imageTask, List imageTaskFiles) { + this.imageTask = imageTask; + this.imageTaskFiles = imageTaskFiles; + } + + public ImageTaskFileList(JSONObject jsonObject) throws JSONException { + ImageTask imageTask = new ImageTask(jsonObject.getString(IMAGE_TASK_ID), + jsonObject.getString(REGION), + (Date) jsonObject.get(IMAGE_DATE) + ); + List imageTaskFiles = new ArrayList<>(); + JSONArray filesJSONArray = jsonObject.getJSONArray(FILES); + for (int i = 0; i < filesJSONArray.length(); i++) { + imageTaskFiles.add(new ImageTaskFile(filesJSONArray.optJSONObject(i))); + } + this.imageTask = imageTask; + this.imageTaskFiles = imageTaskFiles; + } + + public JSONObject toJSON() throws JSONException { + JSONObject jsonObject = new JSONObject(); + jsonObject.put(IMAGE_TASK_ID, imageTask.getTaskId()); + try { + jsonObject.put(REGION, imageTask.getRegion()); + jsonObject.put(IMAGE_DATE, imageTask.getImageDate()); + jsonObject.put(COLLECTION_TIER_NAME, imageTask.getCollectionTierName()); + JSONArray imageTaskFilesJSONArray = new JSONArray(); + for (ImageTaskFile imageTaskFile : imageTaskFiles) { + imageTaskFilesJSONArray.put(imageTaskFile.toJSON()); + } + jsonObject.put(FILES, imageTaskFilesJSONArray); + } catch (JSONException e) { + jsonObject.put(STATUS, UNAVAILABLE); + throw e; + } + return jsonObject; + } +} diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/service/EmailService.java b/src/main/java/org/fogbowcloud/saps/engine/core/service/EmailService.java new file mode 100644 index 000000000..5a472c0e0 --- /dev/null +++ b/src/main/java/org/fogbowcloud/saps/engine/core/service/EmailService.java @@ -0,0 +1,59 @@ +package org.fogbowcloud.saps.engine.core.service; + +import org.fogbowcloud.saps.engine.scheduler.util.SapsPropertiesConstants; +import org.fogbowcloud.saps.notifier.GoogleMail; + +import java.util.Properties; + +/** + * Service to handle Email operations. + */ +public class EmailService { + + /** + * Asynchronously sends an email with specified parameters and default + * no-reply email and password. + * + * @param recipientEmail Recipient email. + * @param title Email title. + * @param message Email message. + */ + public static void sendEmail( + Properties properties, + String recipientEmail, + String title, + String message) { + sendEmail( + properties.getProperty(SapsPropertiesConstants.NO_REPLY_EMAIL), + properties.getProperty(SapsPropertiesConstants.NO_REPLY_PASS), + recipientEmail, + title, + message + ); + } + + /** + * Asynchronously sends an email with specified parameters. + * + * @param username Username of sender account. + * @param password Password of sender account. + * @param recipientEmail Recipient email. + * @param title Email title. + * @param message Email message. + */ + public static void sendEmail( + String username, + String password, + String recipientEmail, + String title, + String message) { + Thread emailThread = new Thread(() -> GoogleMail.Send( + username, + password, + recipientEmail, + title, + message + )); + emailThread.start(); + } +} diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/service/ObjectStoreService.java b/src/main/java/org/fogbowcloud/saps/engine/core/service/ObjectStoreService.java new file mode 100644 index 000000000..d73fa63f0 --- /dev/null +++ b/src/main/java/org/fogbowcloud/saps/engine/core/service/ObjectStoreService.java @@ -0,0 +1,114 @@ +package org.fogbowcloud.saps.engine.core.service; + +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.apache.log4j.Logger; +import org.fogbowcloud.manager.core.plugins.identity.openstack.KeystoneV3IdentityPlugin; +import org.fogbowcloud.manager.occi.model.Token; +import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.fogbowcloud.saps.engine.scheduler.util.SapsPropertiesConstants; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.*; + +/** + * Service that provides operations for communication with Object Store. + */ +public class ObjectStoreService { + + private static final Logger LOGGER = Logger.getLogger(ObjectStoreService.class); + + private static final String HTTPS_SCHEME = "https"; + private static final String PROJECT_ID = "projectId"; + private static final String USER_ID = "userId"; + private static final String PASSWORD = "password"; + private static final String AUTH_URL = "authUrl"; + private static final String PATH_PARAM = "path"; + private static final String X_AUTH_TOKEN = "X-Auth-Token"; + private static final String HTTP_RESPONSE_SEPARATOR = "\n"; + private static final String ARCHIVER_PATH = "archiver/"; + private static final String DATA_OUTPUT_PATH = "/data/output/"; + + /** + * Returns all the paths for every file generated by the {@link ImageTask} + * that had its ID specified. + * + * @param properties Properties that contains Object Store information. + * @param imageTaskId ImageTask's ID. + * @return List of paths. + */ + public static List getImageTaskFilesPaths(Properties properties, String imageTaskId) { + List imageTaskFilesPaths = new ArrayList<>(); + try { + HttpClient client = HttpClients.createDefault(); + HttpGet httpget = prepareObjectStoreRequest(properties, imageTaskId); + HttpResponse response = client.execute(httpget); + imageTaskFilesPaths = parseHttpResponse(response); + } catch (IOException | URISyntaxException e) { + LOGGER.error("Error while retrieving path of files of ImageTask" + + " with ID: " + imageTaskId, e); + } + return imageTaskFilesPaths; + } + + /** + * Parses the given {@link HttpResponse} to a list of Strings. + * + * @param response Response to be parsed. + * @return List of Strings parsed from response. + * @throws IOException + */ + private static List parseHttpResponse(HttpResponse response) throws IOException { + return Arrays.asList(EntityUtils.toString(response.getEntity()).split(HTTP_RESPONSE_SEPARATOR)); + } + + /** + * Prepares a {@link HttpGet} request for the Object Store. + * + * @param properties Properties that contains Object Store information. + * @param imageTaskId ImageTask's ID. + * @return A {@link HttpGet} request. + * @throws URISyntaxException + */ + private static HttpGet prepareObjectStoreRequest(Properties properties, + String imageTaskId) throws URISyntaxException { + String objectStoreHost = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_HOST); + String objectStorePath = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_PATH); + String objectStoreContainer = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_CONTAINER); + String pathParamValue = ARCHIVER_PATH + imageTaskId + DATA_OUTPUT_PATH; + URI uri = new URIBuilder() + .setScheme(HTTPS_SCHEME) + .setHost(objectStoreHost) + .setPath(objectStorePath + "/" + objectStoreContainer) + .addParameter(PATH_PARAM, pathParamValue) + .build(); + LOGGER.debug("Getting list of files for task " + imageTaskId + " from " + uri); + HttpGet httpget = new HttpGet(uri); + Token token = getKeystoneToken(properties); + httpget.addHeader(X_AUTH_TOKEN, token.getAccessId()); + return httpget; + } + + /** + * Returns a Keystone Token for the given properties. + * + * @param properties Properties that contains Object Store information. + * @return Keystone Token. + */ + private static Token getKeystoneToken(Properties properties) { + Map credentials = new HashMap<>(); + credentials.put(PROJECT_ID, properties.getProperty(SapsPropertiesConstants.SWIFT_PROJECT_ID)); + credentials.put(USER_ID, properties.getProperty(SapsPropertiesConstants.SWIFT_USER_ID)); + credentials.put(PASSWORD, properties.getProperty(SapsPropertiesConstants.SWIFT_PASSWORD)); + credentials.put(AUTH_URL, properties.getProperty(SapsPropertiesConstants.SWIFT_AUTH_URL)); + KeystoneV3IdentityPlugin keystone = new KeystoneV3IdentityPlugin(properties); + return keystone.createToken(credentials); + } + +} diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/service/ProcessedImagesService.java b/src/main/java/org/fogbowcloud/saps/engine/core/service/ProcessedImagesService.java new file mode 100644 index 000000000..9ecff7732 --- /dev/null +++ b/src/main/java/org/fogbowcloud/saps/engine/core/service/ProcessedImagesService.java @@ -0,0 +1,146 @@ +package org.fogbowcloud.saps.engine.core.service; + +import org.apache.log4j.Logger; +import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFile; +import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFileList; +import org.fogbowcloud.saps.engine.scheduler.util.SapsPropertiesConstants; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.io.File; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.SignatureException; +import java.util.*; + +/** + * Service the provides operations with processed {@link ImageTask}. + */ +public class ProcessedImagesService { + + private static final Logger LOGGER = Logger.getLogger(ProcessedImagesService.class); + + private static final String TEMP_DIR_URL = "%s?temp_url_sig=%s&temp_url_expires=%s"; + private static final String UNAVAILABLE = "UNAVAILABLE"; + private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1"; + private static final String HTTPS_SCHEME = "https://"; + private static final String FILENAME_QUERY_PARAM = "&filename="; + private static final String GET = "GET"; + private static final String OBJECT_STORE_ENCRIPTION_FORMAT = "%s\n%s\n%s"; + + /** + * Generates a {@link ImageTaskFileList} object containing every file generated + * by the processing of {@link ImageTask} that was specified. + * + * @param properties Properties that contains Object Store information. + * @param imageTask ImageTask that will have its files returned. + * @return A {@link ImageTaskFileList}. + */ + public static ImageTaskFileList generateImageTaskFiles(Properties properties, ImageTask imageTask) { + List filesPaths = ObjectStoreService.getImageTaskFilesPaths(properties, imageTask.getTaskId()); + List imageTaskFiles = new ArrayList<>(); + for (String filePath: filesPaths) { + File file = new File(filePath); + ImageTaskFile imageTaskFile = new ImageTaskFile(filePath, file.getName()); + String imageTaskFileURL; + try { + imageTaskFileURL = generateTempURL(properties, imageTaskFile); + } catch (NoSuchAlgorithmException | SignatureException | InvalidKeyException e) { + LOGGER.error("Failed to generate download link for file " + filePath, e); + imageTaskFileURL = UNAVAILABLE; + } + imageTaskFile.setURL(imageTaskFileURL); + imageTaskFiles.add(imageTaskFile); + } + return new ImageTaskFileList(imageTask, imageTaskFiles); + } + + /** + * Generates a temporary access URL for given {@link ImageTaskFile}. + * + * @param properties Properties that contains Object Store information. + * @param imageTaskFile File to have its access URL generated. + * @return Temporary access URL for given {@link ImageTaskFile}. + */ + private static String generateTempURL(Properties properties, ImageTaskFile imageTaskFile) + throws NoSuchAlgorithmException, SignatureException, InvalidKeyException { + String objectStoreHost = properties + .getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_HOST); + String objectStorePath = properties + .getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_PATH); + String objectStoreContainer = properties + .getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_CONTAINER); + String objectStoreKey = properties + .getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_KEY); + String path = generateTempURLPath( + objectStorePath, + objectStoreContainer, + imageTaskFile.getPath(), + objectStoreKey); + return HTTPS_SCHEME + + objectStoreHost + + path + + FILENAME_QUERY_PARAM + + imageTaskFile.getName(); + } + + /** + * Generates the path of a temporary URL. + * + * @param swiftPath Path to Swift Object Store. + * @param container Container of Swift Object Store. + * @param filePath Path to the file that will be accessed. + * @param key Key of encryption. + * @return Path of a temporary URL. + */ + private static String generateTempURLPath( + String swiftPath, + String container, + String filePath, + String key) + throws NoSuchAlgorithmException, SignatureException, InvalidKeyException { + Formatter objectStoreFormatter = new Formatter(); + String path = swiftPath + "/" + container + "/" + filePath; + objectStoreFormatter.format(OBJECT_STORE_ENCRIPTION_FORMAT, GET, Long.MAX_VALUE, path); + String signature = calculateRFC2104HMAC(objectStoreFormatter.toString(), key); + objectStoreFormatter.close(); + objectStoreFormatter = new Formatter(); + objectStoreFormatter.format(TEMP_DIR_URL, path, signature, Long.MAX_VALUE); + String tempURLPath = objectStoreFormatter.toString(); + objectStoreFormatter.close(); + return tempURLPath; + } + + /** + * Encrypts the given data with the given key. + * + * @param data Data to be encrypted. + * @param key Key of encryption. + * @return The encrypted string. + */ + private static String calculateRFC2104HMAC(String data, String key) + throws SignatureException, NoSuchAlgorithmException, InvalidKeyException { + SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), HMAC_SHA1_ALGORITHM); + Mac mac = Mac.getInstance(HMAC_SHA1_ALGORITHM); + mac.init(signingKey); + return parseToHexString(mac.doFinal(data.getBytes())); + } + + /** + * Parses a array of bytes to a hexadecimal string. + * + * @param bytes Array of bytes to be parsed. + * @return Hexadecimal string parsed from given array of bytes. + */ + private static String parseToHexString(byte[] bytes) { + Formatter formatter = new Formatter(); + for (byte b : bytes) { + formatter.format("%02x", b); + } + String hexString = formatter.toString(); + formatter.close(); + return hexString; + } + +} diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java index 50e43c633..e2e6328bf 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java @@ -1,7 +1,6 @@ package org.fogbowcloud.saps.engine.scheduler.restlet; import java.io.File; -import java.io.IOException; import java.sql.SQLException; import java.text.ParseException; import java.util.Collections; @@ -95,7 +94,7 @@ public Restlet createInboundRoot() { router.attach("/images/{imgName}", ImageResource.class); router.attach("/regions/details", RegionResource.class); router.attach("/regions/search", RegionResource.class); - router.attach("/email", ProcessedImagesResource.class); + router.attach("/email", ImagesEmailResource.class); router.attach("/archivedTasks", ArchivedTasksResource.class); return router; @@ -104,6 +103,10 @@ public Restlet createInboundRoot() { public List getTasks() throws SQLException, ParseException { return submissionDispatcher.getTaskListInDB(); } + + public List getImageTasks(String[] imageTasksIds) throws SQLException { + return submissionDispatcher.getImageTasks(imageTasksIds); + } public List getTasksInState(ImageTaskState imageState) throws SQLException { return this.submissionDispatcher.getTasksInState(imageState); diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java index 5e6318471..89cebc199 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java @@ -1,27 +1,38 @@ package org.fogbowcloud.saps.engine.scheduler.restlet.resource; -import java.util.List; - import org.apache.log4j.Logger; import org.fogbowcloud.saps.engine.core.dispatcher.SubmissionParameters; import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFileList; +import org.fogbowcloud.saps.engine.core.service.ProcessedImagesService; +import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import org.restlet.data.Form; import org.restlet.data.MediaType; import org.restlet.representation.Representation; import org.restlet.representation.StringRepresentation; +import org.restlet.resource.Get; import org.restlet.resource.Post; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + /** - * Responsable for retrieve information about already processed - archived - tasks. + * Responsible for retrieve information about already processed {@link ImageTask}. */ public class ArchivedTasksResource extends BaseResource { private static final Logger LOGGER = Logger.getLogger(ArchivedTasksResource.class); - public ArchivedTasksResource() { - super(); - } + private static final String REQUEST_ATTR_PROCESSED_IMAGES = "images_id[]"; + private static final String IMAGETASKS_IDS_SEPARATOR = ","; + private static final String LIST_OF_IDS_OF_IMAGETASKS_REQUIRED = + "A lista de IDs das ImageTasks é obrigatória."; /** * Returns all processed tasks that matches the execution parameters passed @@ -49,4 +60,63 @@ public Representation getProcessedTasksInInterval(Representation representation) return new StringRepresentation(resObj.toString(), MediaType.APPLICATION_JSON); } + /** + * Returns the files generated by execution of each {@link ImageTask} that + * had its ID specified. + */ + @Get + public Representation getProcessedImageTasksFiles() { + String imageTasksIdsParam = getQuery().getValues(REQUEST_ATTR_PROCESSED_IMAGES); + if (Objects.isNull(imageTasksIdsParam)) { + return new StringRepresentation(LIST_OF_IDS_OF_IMAGETASKS_REQUIRED, MediaType.TEXT_PLAIN); + } + + String[] imageTasksIds = imageTasksIdsParam.split(IMAGETASKS_IDS_SEPARATOR); + LOGGER.info("Recovering files from processed ImageTasks from list of IDs: " + + Arrays.toString(imageTasksIds)); + + List imageTaskFileLists = getImageTaskFileLists(imageTasksIds); + JSONArray imageTaskFileListsJSONArray = parseToJsonArray(imageTaskFileLists); + return new StringRepresentation(imageTaskFileListsJSONArray.toString(), MediaType.APPLICATION_JSON); + } + + /** + * Gets a list of {@link ImageTaskFileList} given the IDs of {@link ImageTask}. + * + * @param imageTasksIds IDs of ImageTasks to have its ImageTaskFileList returned. + * @return List of {@link ImageTaskFileList}. + */ + private List getImageTaskFileLists(String[] imageTasksIds) { + List imageTaskFileLists = new ArrayList<>(); + try { + List imageTasks = application.getImageTasks(imageTasksIds); + imageTaskFileLists = imageTasks.stream() + .map(imageTask -> ProcessedImagesService.generateImageTaskFiles( + application.getProperties(), + imageTask)) + .collect(Collectors.toList()); + } catch (SQLException e) { + LOGGER.error("Error while getting ImageTasks from Catalogue", e); + } + return imageTaskFileLists; + } + + /** + * Parses the specified list of {@link ImageTaskFileList} to a {@link JSONArray}. + * + * @param imageTaskFileLists List of ImageTaskFileList to be parsed. + * @return List of ImageTaskFileList parsed to JSONArray. + */ + private JSONArray parseToJsonArray(List imageTaskFileLists) { + JSONArray resultJSONArray = new JSONArray(); + try { + for (ImageTaskFileList imageTaskFileList: imageTaskFileLists) { + resultJSONArray.put(imageTaskFileList.toJSON()); + } + } catch (JSONException e) { + LOGGER.error("Error while parsing list of ImageTaskFileList to JSONArray", e); + } + return resultJSONArray; + } + } \ No newline at end of file diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java new file mode 100644 index 000000000..8019212d5 --- /dev/null +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java @@ -0,0 +1,220 @@ +package org.fogbowcloud.saps.engine.scheduler.restlet.resource; + +import org.apache.commons.httpclient.HttpStatus; +import org.apache.log4j.Logger; +import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.fogbowcloud.saps.engine.core.model.ImageTaskState; +import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFileList; +import org.fogbowcloud.saps.engine.core.service.EmailService; +import org.fogbowcloud.saps.engine.core.service.ProcessedImagesService; +import org.json.JSONArray; +import org.json.JSONException; +import org.restlet.data.Form; +import org.restlet.data.MediaType; +import org.restlet.ext.json.JsonConverter; +import org.restlet.representation.Representation; +import org.restlet.representation.StringRepresentation; +import org.restlet.resource.ClientResource; +import org.restlet.resource.Post; +import org.restlet.resource.ResourceException; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.*; +import java.util.stream.Collectors; + +/** + * Resource to handle email sending about ImageTasks. + */ +public class ImagesEmailResource extends BaseResource { + + public static final Logger LOGGER = Logger.getLogger(ImagesEmailResource.class); + + private static final int JSON_ARRAY_TO_STRING_FACTOR = 2; + private static final String REQUEST_ATTR_PROCESSED_IMAGES = "images_id[]"; + private static final String PROCESSED_TASKS_URN = "/archivedTasks"; + private static final String PROCESSED_IMAGES_EMAIL_TITLE = "[SAPS] Filter results"; + private static final String SUCCESSFUL_EMAIL_SENDING_FEEDBACK = + "Email será enviado em breve."; + private static final String UNSUCCESSFUL_EMAIL_SENDING_FEEDBACK = + "Houve um erro ao enviar o email, tente novamente."; + + /** + * Sends information of files generated by execution of ImageTasks that + * have had its IDs specified via HTTP body. + * + * @param representation Representation containing data sent via HTTP body. + * @return Representation with feedback of email sending. + */ + @Post + public Representation sendProcessedImagesToEmail(Representation representation) { + Form form = new Form(representation); + String userEmail = form.getFirstValue(UserResource.REQUEST_ATTR_USER_EMAIL, true); + String userPass = form.getFirstValue(UserResource.REQUEST_ATTR_USERPASS, true); + if (!authenticateUser(userEmail, userPass) || userEmail.equals("anonymous")) { + throw new ResourceException(HttpStatus.SC_UNAUTHORIZED); + } + + String[] imageTasksIds = form.getValuesArray(REQUEST_ATTR_PROCESSED_IMAGES, true); + try { + List allImageTasksFileList = getImageTaskFileLists(imageTasksIds); + String emailMessage = generateEmailMessage(allImageTasksFileList); + Properties properties = application.getProperties(); + EmailService.sendEmail( + properties, + userEmail, + PROCESSED_IMAGES_EMAIL_TITLE, + emailMessage); + return new StringRepresentation(SUCCESSFUL_EMAIL_SENDING_FEEDBACK, MediaType.TEXT_PLAIN); + } catch (Throwable t) { + LOGGER.error("Error while sending email of processed images", t); + } + return new StringRepresentation(UNSUCCESSFUL_EMAIL_SENDING_FEEDBACK, MediaType.TEXT_PLAIN); + } + + /** + * Generates a email message from list of {@link ImageTaskFileList}. + * + * @param imageTasksFileLists List of ImageTaskFileList. + * @return Email message. + */ + private String generateEmailMessage(List imageTasksFileLists) throws JSONException { + JSONArray emailMessageJSON = generateEmailMessageJSON(imageTasksFileLists); + try { + return emailMessageJSON.toString(JSON_ARRAY_TO_STRING_FACTOR); + } catch (JSONException e) { + LOGGER.error("Error while generating String email message from JSON", e); + throw e; + } + } + + /** + * Generates a email message in form of JSONArray from list of + * {@link ImageTaskFileList}. + * + * @param imageTasksFileLists List of ImageTaskFileList. + * @return Email message in form of a JSONArray. + */ + private JSONArray generateEmailMessageJSON(List imageTasksFileLists) { + JSONArray emailMessageJSON = new JSONArray(); + try { + for (ImageTaskFileList imageTasksFileList: imageTasksFileLists) { + emailMessageJSON.put(imageTasksFileList.toJSON()); + } + } catch (JSONException e) { + LOGGER.error("Error while parsing list of files to JSON of ImageTask", e); + } + return emailMessageJSON; + } + + /** + * Returns a list of {@link ImageTaskFileList}, each one of these for each + * {@link ImageTask} that had its ID in specified list of IDs. + * + * @param imageTasksIds List of IDs of ImageTasks. + * @return List of ImageTaskFileList. + */ + private List getImageTaskFileLists(String[] imageTasksIds) throws SQLException { + try { + List imageTasks = application.getImageTasks(imageTasksIds); + List localImageTasksFileLists = getLocalImageTaskFileLists(imageTasks); + List remoteImageTasksFileList = getRemoteImageTaskFileLists(imageTasks); + List allImageTasksFileList = new ArrayList<>(); + allImageTasksFileList.addAll(localImageTasksFileLists); + allImageTasksFileList.addAll(remoteImageTasksFileList); + return allImageTasksFileList; + } catch (SQLException e) { + LOGGER.error("Error while getting list of ImageTaskFileList to send email", e); + throw e; + } + } + + /** + * Returns a list of {@link ImageTaskFileList}, each one of these for each + * local {@link ImageTask} from specified list of ImageTasks. + * + * @param imageTasks List of ImageTasks containing local ImageTasks. + * @return List of ImageTaskFileList. + */ + private List getLocalImageTaskFileLists(List imageTasks) { + List localImageTasks = imageTasks.stream() + .filter(imageTask -> imageTask.getState().equals(ImageTaskState.ARCHIVED)) + .collect(Collectors.toList()); + return localImageTasks.stream() + .map(imageTask -> ProcessedImagesService.generateImageTaskFiles( + application.getProperties(), + imageTask)) + .collect(Collectors.toList()); + } + + /** + * Returns a list of {@link ImageTaskFileList}, each one of these for each + * remote {@link ImageTask} from specified list of ImageTasks. + * + * @param imageTasks List of ImageTasks containing remote ImageTasks. + * @return List of ImageTaskFileList. + */ + private List getRemoteImageTaskFileLists(List imageTasks) { + List remoteImageTasks = imageTasks + .stream() + .filter(imageTask -> imageTask.getState().equals(ImageTaskState.REMOTELY_ARCHIVED)) + .collect(Collectors.toList()); + Map> remoteImageTaskGroupedBySAPSInstanceURL = remoteImageTasks + .stream() + .collect(Collectors.groupingBy(ImageTask::getFederationMember)); + return remoteImageTaskGroupedBySAPSInstanceURL + .entrySet() + .stream() + .map(this::getImageTasksFilesFromSAPSNeighbor) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + } + + /** + * Returns a list of {@link ImageTaskFileList} for SAPS neighbor that had + * its URL specified. + * + * @param SAPSNeighborAndImageTasks {@link Map.Entry} that have a SAPS + * neighbor URL as key and a list of + * ImageTasks as value. + * @return List of {@link ImageTaskFileList}. + */ + private List getImageTasksFilesFromSAPSNeighbor( + Map.Entry> SAPSNeighborAndImageTasks) { + String SAPSNeighborURL = SAPSNeighborAndImageTasks.getKey(); + List imageTasks = SAPSNeighborAndImageTasks.getValue(); + List imageTasksIds = imageTasks.stream() + .map(ImageTask::getTaskId) + .collect(Collectors.toList()); + List imageTaskFileLists = new ArrayList<>(); + try { + ClientResource clientResource = new ClientResource(SAPSNeighborURL + PROCESSED_TASKS_URN); + for (String imageTaskId: imageTasksIds) { + clientResource.addQueryParameter(REQUEST_ATTR_PROCESSED_IMAGES, imageTaskId); + } + Representation response = clientResource.get(MediaType.APPLICATION_JSON); + imageTaskFileLists = extractImageTaskFileLists(response); + } catch (Throwable t) { + LOGGER.error("Error while getting ImageTaskFileList from SAPS Neighbor.", t); + } + return imageTaskFileLists; + } + + /** + * Extract a list of {@link ImageTaskFileList} from specified response object. + * + * @param response Response containing a list of ImageTaskFileList. + * @return List of ImageTaskFileList. + */ + private List extractImageTaskFileLists(Representation response) + throws IOException, JSONException { + JsonConverter jsonConverter = new JsonConverter(); + JSONArray responseJson = jsonConverter.toObject(response, JSONArray.class, null); + List imageTaskFileLists = new ArrayList<>(); + for (int i = 0; i < responseJson.length(); i++) { + imageTaskFileLists.add(new ImageTaskFileList(responseJson.optJSONObject(i))); + } + return imageTaskFileLists; + } + +} diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ProcessedImagesResource.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ProcessedImagesResource.java deleted file mode 100644 index f424e4345..000000000 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ProcessedImagesResource.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.fogbowcloud.saps.engine.scheduler.restlet.resource; - -import org.apache.commons.httpclient.HttpStatus; -import org.apache.log4j.Logger; -import org.fogbowcloud.saps.engine.scheduler.util.ProcessedImagesEmailBuilder; -import org.restlet.data.Form; -import org.restlet.data.MediaType; -import org.restlet.representation.Representation; -import org.restlet.representation.StringRepresentation; -import org.restlet.resource.Post; -import org.restlet.resource.ResourceException; - -import java.util.Arrays; -import java.util.Properties; - -public class ProcessedImagesResource extends BaseResource { - - public static final Logger LOGGER = Logger.getLogger(ProcessedImagesResource.class); - - private static final String REQUEST_ATTR_PROCESSED_IMAGES = "images_id[]"; - - @Post - public Representation sendProcessedImagesToEmail(Representation representation) { - Form form = new Form(representation); - - String userEmail = form.getFirstValue(UserResource.REQUEST_ATTR_USER_EMAIL, true); - String userPass = form.getFirstValue(UserResource.REQUEST_ATTR_USERPASS, true); - if (!authenticateUser(userEmail, userPass) || userEmail.equals("anonymous")) { - throw new ResourceException(HttpStatus.SC_UNAUTHORIZED); - } - - String[] imageIds = form.getValuesArray(REQUEST_ATTR_PROCESSED_IMAGES, true); - Properties properties = application.getProperties(); - - ProcessedImagesEmailBuilder emailBuilder = new ProcessedImagesEmailBuilder( - application, - properties, - userEmail, - Arrays.asList(imageIds) - ); - - Thread thread = new Thread(emailBuilder); - thread.start(); - - return new StringRepresentation("Email será enviado em breve.", MediaType.TEXT_PLAIN); - } -} diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java deleted file mode 100644 index 2e9859d8f..000000000 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java +++ /dev/null @@ -1,297 +0,0 @@ -package org.fogbowcloud.saps.engine.scheduler.util; - -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; -import org.apache.log4j.Logger; -import org.fogbowcloud.manager.core.plugins.identity.openstack.KeystoneV3IdentityPlugin; -import org.fogbowcloud.manager.occi.model.Token; -import org.fogbowcloud.saps.engine.core.model.ImageTask; -import org.fogbowcloud.saps.engine.scheduler.restlet.DatabaseApplication; -import org.fogbowcloud.saps.notifier.GoogleMail; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; -import javax.mail.MessagingException; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; -import java.security.SignatureException; -import java.sql.SQLException; -import java.util.*; - -public class ProcessedImagesEmailBuilder implements Runnable { - - private static final Logger LOGGER = Logger.getLogger(ProcessedImagesEmailBuilder.class); - - private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1"; - private static final String UNAVAILABLE = "UNAVAILABLE"; - private static final String TEMP_DIR_URL = "%s?temp_url_sig=%s&temp_url_expires=%s"; - - private static final String PROJECT_ID = "projectId"; - private static final String PASSWORD = "password"; - private static final String USER_ID = "userId"; - private static final String AUTH_URL = "authUrl"; - private static final String TASK_ID = "taskId"; - private static final String REGION = "region"; - private static final String COLLECTION_TIER_NAME = "collectionTierName"; - private static final String IMAGE_DATE = "imageDate"; - private static final String NAME = "name"; - private static final String URL = "url"; - private static final String FILES = "files"; - private static final String STATUS = "status"; - - private DatabaseApplication application; - private Properties properties; - private String userEmail; - private List images; - - public ProcessedImagesEmailBuilder( - DatabaseApplication databaseApplication, - Properties properties, - String userEmail, - List images - ) { - this.application = databaseApplication; - this.properties = properties; - this.userEmail = userEmail; - this.images = images; - } - - @Override - public void run() { - StringBuilder builder = new StringBuilder(); - builder.append("Creating email for user "); - builder.append(userEmail); - builder.append(" with images:\n"); - for (String str: images) { - builder.append(str).append("\n"); - } - LOGGER.info(builder.toString()); - - StringBuilder errorBuilder = new StringBuilder(); - JSONArray tasklist = generateAllTasksJsons(errorBuilder); - sendTaskEmail(errorBuilder, tasklist); - sendErrorEmail(errorBuilder); - } - - JSONArray generateAllTasksJsons(StringBuilder errorBuilder) { - JSONArray tasklist = new JSONArray(); - for (String str: images) { - try { - tasklist.put(generateTaskEmailJson(properties, str)); - } catch (IOException | URISyntaxException | IllegalArgumentException e) { - LOGGER.error("Failed to fetch image from object store.", e); - errorBuilder - .append("Failed to fetch image from object store.").append("\n") - .append(ExceptionUtils.getStackTrace(e)).append("\n"); - try { - JSONObject task = new JSONObject(); - task.put(TASK_ID, str); - task.put(STATUS, UNAVAILABLE); - tasklist.put(task); - } catch (JSONException e1) { - LOGGER.error("Failed to create UNAVAILABLE task json.", e); - } - } catch (SQLException e) { - LOGGER.error("Failed to fetch image from database.", e); - errorBuilder - .append("Failed to fetch image from database.").append("\n") - .append(ExceptionUtils.getStackTrace(e)).append("\n"); - try { - JSONObject task = new JSONObject(); - task.put(TASK_ID, str); - task.put(STATUS, UNAVAILABLE); - tasklist.put(task); - } catch (JSONException e1) { - LOGGER.error("Failed to create UNAVAILABLE task json.", e); - } - } catch (JSONException e) { - LOGGER.error("Failed to create task json.", e); - errorBuilder - .append("Failed to create task json.").append("\n") - .append(ExceptionUtils.getStackTrace(e)).append("\n"); - try { - JSONObject task = new JSONObject(); - task.put(TASK_ID, str); - task.put(STATUS, UNAVAILABLE); - tasklist.put(task); - } catch (JSONException e1) { - LOGGER.error("Failed to create UNAVAILABLE task json.", e); - } - } - } - return tasklist; - } - - private void sendTaskEmail(StringBuilder errorBuilder, JSONArray tasklist) { - try { - GoogleMail.Send( - properties.getProperty(SapsPropertiesConstants.NO_REPLY_EMAIL), - properties.getProperty(SapsPropertiesConstants.NO_REPLY_PASS), - userEmail, - "[SAPS] Filter results", - tasklist.toString(2) - ); - } catch (MessagingException | JSONException e) { - LOGGER.error("Failed to send email with images download links.", e); - errorBuilder - .append("Failed to send email with images download links.").append("\n") - .append(ExceptionUtils.getStackTrace(e)).append("\n"); - } - } - - private void sendErrorEmail(StringBuilder errorBuilder) { - if (!errorBuilder.toString().isEmpty()) { - try { - GoogleMail.Send( - properties.getProperty(SapsPropertiesConstants.NO_REPLY_EMAIL), - properties.getProperty(SapsPropertiesConstants.NO_REPLY_PASS), - "sebal.no.reply@gmail.com", - "[SAPS] Errors during image temporary link creation", - errorBuilder.toString() - ); - } catch (MessagingException e) { - LOGGER.error("Failed to send email with errors to admins.", e); - } - } - } - - private String toHexString(byte[] bytes) { - Formatter formatter = new Formatter(); - - for (byte b : bytes) { - formatter.format("%02x", b); - } - - String hexString = formatter.toString(); - formatter.close(); - - return hexString; - } - - private String calculateRFC2104HMAC(String data, String key) - throws SignatureException, NoSuchAlgorithmException, InvalidKeyException - { - SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), HMAC_SHA1_ALGORITHM); - Mac mac = Mac.getInstance(HMAC_SHA1_ALGORITHM); - mac.init(signingKey); - return toHexString(mac.doFinal(data.getBytes())); - } - - String generateTempURL(String swiftPath, String container, String filePath, String key) - throws NoSuchAlgorithmException, SignatureException, InvalidKeyException { - String path = swiftPath + "/" + container + "/" + filePath; - - Formatter objectStoreFormatter = new Formatter(); - objectStoreFormatter.format("%s\n%s\n%s", "GET", Long.MAX_VALUE, path); - String signature = calculateRFC2104HMAC(objectStoreFormatter.toString(), key); - objectStoreFormatter.close(); - - objectStoreFormatter = new Formatter(); - objectStoreFormatter.format( - TEMP_DIR_URL, - path, - signature, - Long.MAX_VALUE - ); - String res = objectStoreFormatter.toString(); - objectStoreFormatter.close(); - - return res; - } - - JSONObject generateTaskEmailJson(Properties properties, String imageid) - throws URISyntaxException, IOException, SQLException, JSONException { - JSONObject result = new JSONObject(); - - String objectStoreHost = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_HOST); - String objectStorePath = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_PATH); - String objectStoreContainer = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_CONTAINER); - String objectStoreKey = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_KEY); - - ImageTask task = application.getTask(imageid); - - List files = getTaskFilesFromObjectStore( - properties, objectStoreHost, objectStorePath, objectStoreContainer, task - ); - - result.put(TASK_ID, task.getTaskId()); - result.put(REGION, task.getRegion()); - result.put(COLLECTION_TIER_NAME, task.getCollectionTierName()); - result.put(IMAGE_DATE, task.getImageDate()); - - JSONArray filelist = new JSONArray(); - for (String str: files) { - File f = new File(str); - String fileName = f.getName(); - try { - JSONObject fileobject = new JSONObject(); - fileobject.put(NAME, fileName); - fileobject.put(URL, "https://" + objectStoreHost + generateTempURL( - objectStorePath, - objectStoreContainer, - str, - objectStoreKey) + "&filename=" + fileName - ); - filelist.put(fileobject); - } catch (NoSuchAlgorithmException | SignatureException | InvalidKeyException e) { - LOGGER.error("Failed to generate download link for file " + str, e); - try { - JSONObject fileobject = new JSONObject(); - fileobject.put(NAME, fileName); - fileobject.put(URL, UNAVAILABLE); - filelist.put(fileobject); - } catch (JSONException e1) { - LOGGER.error("Failed to create UNAVAILABLE temp url json.", e); - } - } - } - result.put(FILES, filelist); - - return result; - } - - List getTaskFilesFromObjectStore(Properties properties, String objectStoreHost, String objectStorePath, String objectStoreContainer, ImageTask task) throws URISyntaxException, IOException { - Token token = getKeystoneToken(properties); - - HttpClient client = HttpClients.createDefault(); - HttpGet httpget = prepObjectStoreRequest(objectStoreHost, objectStorePath, objectStoreContainer, task, token); - HttpResponse response = client.execute(httpget); - - return Arrays.asList(EntityUtils.toString(response.getEntity()).split("\n")); - } - - private HttpGet prepObjectStoreRequest(String objectStoreHost, String objectStorePath, String objectStoreContainer, ImageTask task, Token token) throws URISyntaxException { - URI uri = new URIBuilder() - .setScheme("https") - .setHost(objectStoreHost) - .setPath(objectStorePath + "/" + objectStoreContainer) - .addParameter("path", "archiver/" + task.getTaskId() + "/data/output/") - .build(); - LOGGER.debug("Getting list of files for task " + task.getTaskId() + " from " + uri); - HttpGet httpget = new HttpGet(uri); - httpget.addHeader("X-Auth-Token", token.getAccessId()); - return httpget; - } - - private Token getKeystoneToken(Properties properties) { - KeystoneV3IdentityPlugin keystone = new KeystoneV3IdentityPlugin(properties); - Map credentials = new HashMap<>(); - credentials.put(AUTH_URL, properties.getProperty(SapsPropertiesConstants.SWIFT_AUTH_URL)); - credentials.put(PROJECT_ID, properties.getProperty(SapsPropertiesConstants.SWIFT_PROJECT_ID)); - credentials.put(USER_ID, properties.getProperty(SapsPropertiesConstants.SWIFT_USER_ID)); - credentials.put(PASSWORD, properties.getProperty(SapsPropertiesConstants.SWIFT_PASSWORD)); - return keystone.createToken(credentials); - } -} diff --git a/src/test/java/org/fogbowcloud/saps/engine/scheduler/util/TestProcessedImagesEmailBuilder.java b/src/test/java/org/fogbowcloud/saps/engine/scheduler/util/TestProcessedImagesEmailBuilder.java deleted file mode 100644 index 9b0b08aeb..000000000 --- a/src/test/java/org/fogbowcloud/saps/engine/scheduler/util/TestProcessedImagesEmailBuilder.java +++ /dev/null @@ -1,280 +0,0 @@ -package org.fogbowcloud.saps.engine.scheduler.util; - -import org.fogbowcloud.saps.engine.core.model.ImageTask; -import org.fogbowcloud.saps.engine.core.model.ImageTaskState; -import org.fogbowcloud.saps.engine.scheduler.restlet.DatabaseApplication; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Matchers; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; -import java.security.SignatureException; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Properties; - -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; - -public class TestProcessedImagesEmailBuilder { - - private static final String IMAGE_TASK_ID = "task_id_101010101010"; - private static final String UNAVAILABLE = "UNAVAILABLE"; - - @Test - public void testGenerateTaskEmailJson() throws SQLException, IOException, URISyntaxException, JSONException { - Properties properties = getProperties(); - ImageTask imageTask = getImageTask(); - List imageIds = getImageIdList(); - DatabaseApplication application = getDatabaseApplication(imageTask); - ProcessedImagesEmailBuilder emailBuilder = getProcessedImagesEmailBuilder(properties, imageIds, application); - - List imageFiles = getImageFileList(); - doReturn(imageFiles).when(emailBuilder).getTaskFilesFromObjectStore( - any(Properties.class), anyString(), anyString(), anyString(), any(ImageTask.class) - ); - - JSONObject jsonReturn; - jsonReturn = emailBuilder.generateTaskEmailJson(properties, IMAGE_TASK_ID); - - Assert.assertEquals(imageTask.getTaskId(), jsonReturn.getString("taskId")); - Assert.assertEquals(imageTask.getRegion(), jsonReturn.getString("region")); - Assert.assertEquals(imageTask.getImageDate().toString(), jsonReturn.getString("imageDate")); - Assert.assertEquals(imageTask.getCollectionTierName(), jsonReturn.getString("collectionTierName")); - - JSONArray files = jsonReturn.getJSONArray("files"); - Assert.assertEquals(imageFiles.size(), files.length()); - for (int i = 0; i < files.length(); i++) { - boolean missing = true; - for (String imageFile : imageFiles) { - JSONObject f = files.getJSONObject(i); - String name = f.getString("name"); - if (name.equals(imageFile)) { - String url = f.getString("url"); - System.out.println(name + ": " + url); - checkUrlAvailable(properties, name, url); - missing = false; - break; - } - } - if (missing) { - Assert.fail("One of the files is missing"); - } - } - } - - @Test - public void testGenerateTaskEmailJsonErrorGeneratingURL() throws SQLException, IOException, URISyntaxException, NoSuchAlgorithmException, SignatureException, InvalidKeyException, JSONException { - Properties properties = getProperties(); - ImageTask imageTask = getImageTask(); - List imageIds = getImageIdList(); - DatabaseApplication application = getDatabaseApplication(imageTask); - ProcessedImagesEmailBuilder emailBuilder = getProcessedImagesEmailBuilder(properties, imageIds, application); - - List imageFiles = getImageFileList(); - String failingFile = imageFiles.get(3); - - doReturn(imageFiles).when(emailBuilder).getTaskFilesFromObjectStore( - any(Properties.class), anyString(), anyString(), anyString(), any(ImageTask.class) - ); - doThrow(SignatureException.class).when(emailBuilder).generateTempURL( - anyString(), - anyString(), - Matchers.eq(failingFile), - anyString() - ); - - JSONObject jsonReturn; - jsonReturn = emailBuilder.generateTaskEmailJson(properties, IMAGE_TASK_ID); - - Assert.assertEquals(imageTask.getTaskId(), jsonReturn.getString("taskId")); - Assert.assertEquals(imageTask.getRegion(), jsonReturn.getString("region")); - Assert.assertEquals(imageTask.getImageDate().toString(), jsonReturn.getString("imageDate")); - Assert.assertEquals(imageTask.getCollectionTierName(), jsonReturn.getString("collectionTierName")); - - JSONArray files = jsonReturn.getJSONArray("files"); - Assert.assertEquals(imageFiles.size(), files.length()); - for (int i = 0; i < files.length(); i++) { - boolean missing = true; - for (String imageFile : imageFiles) { - JSONObject f = files.getJSONObject(i); - String name = f.getString("name"); - if (name.equals(imageFile)) { - String url = f.getString("url"); - System.out.println(name + ": " + url); - if (name.equals(failingFile)) { - Assert.assertEquals(UNAVAILABLE, url); - } else { - checkUrlAvailable(properties, name, url); - } - missing = false; - break; - } - } - if (missing) { - Assert.fail("One of the files is missing"); - } - } - } - - @Test - public void testGenerateTaskEmailJsonErrorOnDatabase() throws SQLException, IOException, URISyntaxException, NoSuchAlgorithmException, SignatureException, InvalidKeyException, JSONException { - Properties properties = getProperties(); - ImageTask imageTask = getImageTask(); - List imageIds = getImageIdList(); - DatabaseApplication application = getDatabaseApplication(imageTask); - ProcessedImagesEmailBuilder emailBuilder = getProcessedImagesEmailBuilder(properties, imageIds, application); - - try { - emailBuilder.generateTaskEmailJson(properties, "FAIL_TASK"); - Assert.fail("Should throw exception"); - } catch (SQLException e) { - e.printStackTrace(); - } - } - - @Test - public void testGenerateAllTasksJsons() throws SQLException, JSONException, IOException, URISyntaxException { - Properties properties = getProperties(); - ImageTask imageTask = getImageTask(); - List imageIds = getImageIdList(); - imageIds.add("FAIL_TASK"); - DatabaseApplication application = getDatabaseApplication(imageTask); - ProcessedImagesEmailBuilder emailBuilder = getProcessedImagesEmailBuilder(properties, imageIds, application); - - List imageFiles = getImageFileList(); - doReturn(imageFiles).when(emailBuilder).getTaskFilesFromObjectStore( - any(Properties.class), anyString(), anyString(), anyString(), any(ImageTask.class) - ); - - StringBuilder errorBuilder = new StringBuilder(); - JSONArray tasks = emailBuilder.generateAllTasksJsons(errorBuilder); - for (int i = 0; i < tasks.length(); i++) { - JSONObject obj = tasks.getJSONObject(i); - boolean missing = true; - for (String tid: imageIds) { - String name = obj.getString("taskId"); - if (name.equals(tid)) { - System.out.println(obj.toString(2)); - String status = obj.optString("status"); - if (name.equals("FAIL_TASK")) { - Assert.assertEquals(UNAVAILABLE, status); - } else { - Assert.assertNotEquals(UNAVAILABLE, status); - } - missing = false; - break; - } - } - if (missing) { - Assert.fail("One of the tasks is missing"); - } - } - } - - private DatabaseApplication getDatabaseApplication(final ImageTask imageTask) throws SQLException { - DatabaseApplication application = Mockito.mock(DatabaseApplication.class); - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock invocationOnMock) throws Throwable { - Object[] args = invocationOnMock.getArguments(); - String imageId = (String)args[0]; - if (imageId.equals(imageTask.getTaskId())) return imageTask; - else throw new SQLException(); - } - }).when(application).getTask(anyString()); - return application; - } - - private Properties getProperties() { - Properties properties = new Properties(); - properties.setProperty(SapsPropertiesConstants.SWIFT_AUTH_URL, "https://cloud.lsd.ufcg.edu.br:5000"); - properties.setProperty(SapsPropertiesConstants.SWIFT_PROJECT_ID, "3324431f606d4a74a060cf78c16fcb21"); - properties.setProperty(SapsPropertiesConstants.SWIFT_USER_ID, "3e57892203271c195f5d473fc84f484b8062103275ce6ad6e7bcd1baedf70d5c"); - properties.setProperty(SapsPropertiesConstants.SWIFT_PASSWORD, "123456"); - properties.setProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_HOST, "cloud.lsd.ufcg.edu.br:8080"); - properties.setProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_PATH, "/swift/v1"); - properties.setProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_CONTAINER, "lsd_deploy"); - properties.setProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_KEY, "qnjcKtALnbqHkOM0f0J9I8TWzANCI7Qj8"); - properties.setProperty( - SapsPropertiesConstants.NO_REPLY_EMAIL, "sebal.no.reply@gmail.com" - ); - properties.setProperty( - SapsPropertiesConstants.NO_REPLY_PASS, "noREsebal16" - ); - return properties; - } - - private List getImageIdList() { - List imageIds = new ArrayList<>(); - imageIds.add(IMAGE_TASK_ID); - return imageIds; - } - - private ImageTask getImageTask() { - return new ImageTask( - IMAGE_TASK_ID, - "landsat-7", - "263_065", - new Date(), - "NA", - ImageTaskState.ARCHIVED, - "NA", - 0, - "NA", - "Default", - "Default", - "Default", - "NA", - "NA", - new Timestamp(System.nanoTime()), - new Timestamp(System.nanoTime()), - "NA", - "NA" - ); - } - - private ProcessedImagesEmailBuilder getProcessedImagesEmailBuilder(Properties properties, List imageIds, DatabaseApplication application) { - return Mockito.spy( - new ProcessedImagesEmailBuilder( - application, properties, "sebal.no.replay@gmail.com", imageIds - ) - ); - } - - private void checkUrlAvailable(Properties properties, String name, String url) { - Assert.assertNotEquals(UNAVAILABLE, url); - Assert.assertTrue(url.contains(properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_HOST))); - Assert.assertTrue(url.contains(properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_PATH))); - Assert.assertTrue(url.contains(properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_CONTAINER))); - Assert.assertTrue(url.contains(name)); - Assert.assertTrue(url.contains("?temp_url_sig")); - Assert.assertTrue(url.contains("&temp_url_expires")); - Assert.assertTrue(url.contains("&filename="+name)); - } - - private List getImageFileList() { - List imageFiles = new ArrayList<>(); - imageFiles.add("file1"); - imageFiles.add("file2"); - imageFiles.add("file3"); - imageFiles.add("file4"); - return imageFiles; - } - -}