diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/database/ImageDataStore.java b/src/main/java/org/fogbowcloud/saps/engine/core/database/ImageDataStore.java index bf3544325..af57b01a2 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/database/ImageDataStore.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/database/ImageDataStore.java @@ -126,4 +126,6 @@ public List getProcessedImages( String inputGathering, String inputPreprocessing, String algorithmExecution) throws SQLException; + + List getImageTasks(String[] imageTasksIds) throws SQLException; } diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/database/JDBCImageDataStore.java b/src/main/java/org/fogbowcloud/saps/engine/core/database/JDBCImageDataStore.java index aff7d1f4f..396fa0a81 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/database/JDBCImageDataStore.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/database/JDBCImageDataStore.java @@ -1,17 +1,14 @@ package org.fogbowcloud.saps.engine.core.database; +import java.sql.Array; import java.sql.Connection; +import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.dbcp2.BasicDataSource; @@ -1020,6 +1017,27 @@ public List getAllTasks() throws SQLException { } } + private static final String SELECT_IMAGES_IN_ID_LIST_SQL = "SELECT * FROM " + IMAGE_TABLE_NAME + + " WHERE " + TASK_ID_COL + " IN (?)"; + + @Override + public List getImageTasks(String[] imageTasksIds) throws SQLException { + PreparedStatement statement = null; + Connection conn = null; + try { + conn = getConnection(); + statement = conn.prepareStatement(SELECT_IMAGES_IN_ID_LIST_SQL); + Array imageTasksIdsArray = conn.createArrayOf(JDBCType.VARCHAR.getName(), imageTasksIds); + statement.setArray(1, imageTasksIdsArray); + statement.setQueryTimeout(300); + statement.execute(); + ResultSet rs = statement.getResultSet(); + return extractImageTaskFrom(rs); + } finally { + close(statement, conn); + } + } + private static final String SELECT_USER_SQL = "SELECT * FROM " + USERS_TABLE_NAME + " WHERE " + USER_EMAIL_COL + " = ?"; diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java index 4e18161cf..f90771223 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcher.java @@ -41,4 +41,6 @@ void removeUserNotification(String submissionId, String taskId, String userEmail boolean isUserNotifiable(String userEmail) throws SQLException; List searchProcessedTasks(SubmissionParameters submissionParameters); + + List getImageTasks(String[] imageTasksIds) throws SQLException; } diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherImpl.java b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherImpl.java index a189c57f8..5f59886b6 100755 --- a/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherImpl.java +++ b/src/main/java/org/fogbowcloud/saps/engine/core/dispatcher/SubmissionDispatcherImpl.java @@ -269,6 +269,11 @@ public List getTaskListInDB() throws SQLException, ParseException { return imageStore.getAllTasks(); } + @Override + public List getImageTasks(String[] imageTasksIds) throws SQLException { + return imageStore.getImageTasks(imageTasksIds); + } + @Override public List getUsersToNotify() throws SQLException { List wards = imageStore.getUsersToNotify(); diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java new file mode 100644 index 000000000..f6d2c0e43 --- /dev/null +++ b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFile.java @@ -0,0 +1,48 @@ +package org.fogbowcloud.saps.engine.core.pojo; + +import org.fogbowcloud.saps.engine.core.model.ImageTask; + +/** + * Holds the information of a file generated by the processing of a {@link ImageTask}. + */ +public class ImageTaskFile { + + private String path; + private String name; + private String URL; + + public ImageTaskFile(String path, String name) { + this(path, name, null); + } + + public ImageTaskFile(String path, String name, String URL) { + this.path = path; + this.name = name; + this.URL = URL; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getURL() { + return URL; + } + + public void setURL(String URL) { + this.URL = URL; + } + +} diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java new file mode 100644 index 000000000..06d44ba21 --- /dev/null +++ b/src/main/java/org/fogbowcloud/saps/engine/core/pojo/ImageTaskFileList.java @@ -0,0 +1,54 @@ +package org.fogbowcloud.saps.engine.core.pojo; + +import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.util.List; + +/** + * Represents a list of {@link ImageTaskFile} for a specific {@link ImageTask}. + */ +public class ImageTaskFileList { + + private static final String IMAGE_TASK_ID = "imageTaskId"; + private static final String REGION = "region"; + private static final String COLLECTION_TIER_NAME = "collectionTierName"; + private static final String IMAGE_DATE = "imageDate"; + private static final String NAME = "name"; + private static final String URL = "url"; + private static final String FILES = "files"; + private static final String STATUS = "status"; + private static final String UNAVAILABLE = "UNAVAILABLE"; + + private ImageTask imageTask; + private List imageTaskFiles; + + public ImageTaskFileList(ImageTask imageTask, List imageTaskFiles) { + this.imageTask = imageTask; + this.imageTaskFiles = imageTaskFiles; + } + + public JSONObject toJSON() throws JSONException { + JSONObject jsonObject = new JSONObject(); + jsonObject.put(IMAGE_TASK_ID, imageTask.getTaskId()); + try { + jsonObject.put(REGION, imageTask.getRegion()); + jsonObject.put(COLLECTION_TIER_NAME, imageTask.getCollectionTierName()); + jsonObject.put(IMAGE_DATE, imageTask.getImageDate()); + JSONArray filesJSONArray = new JSONArray(); + for (ImageTaskFile imageTaskFile : imageTaskFiles) { + JSONObject imageTaskFileJSONObject = new JSONObject(); + imageTaskFileJSONObject.put(NAME, imageTaskFile.getName()); + imageTaskFileJSONObject.put(URL, imageTaskFile.getURL()); + filesJSONArray.put(imageTaskFileJSONObject); + } + jsonObject.put(FILES, filesJSONArray); + } catch (JSONException e) { + jsonObject.put(STATUS, UNAVAILABLE); + throw e; + } + return jsonObject; + } +} diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/service/ObjectStoreService.java b/src/main/java/org/fogbowcloud/saps/engine/core/service/ObjectStoreService.java new file mode 100644 index 000000000..d73fa63f0 --- /dev/null +++ b/src/main/java/org/fogbowcloud/saps/engine/core/service/ObjectStoreService.java @@ -0,0 +1,114 @@ +package org.fogbowcloud.saps.engine.core.service; + +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.apache.log4j.Logger; +import org.fogbowcloud.manager.core.plugins.identity.openstack.KeystoneV3IdentityPlugin; +import org.fogbowcloud.manager.occi.model.Token; +import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.fogbowcloud.saps.engine.scheduler.util.SapsPropertiesConstants; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.*; + +/** + * Service that provides operations for communication with Object Store. + */ +public class ObjectStoreService { + + private static final Logger LOGGER = Logger.getLogger(ObjectStoreService.class); + + private static final String HTTPS_SCHEME = "https"; + private static final String PROJECT_ID = "projectId"; + private static final String USER_ID = "userId"; + private static final String PASSWORD = "password"; + private static final String AUTH_URL = "authUrl"; + private static final String PATH_PARAM = "path"; + private static final String X_AUTH_TOKEN = "X-Auth-Token"; + private static final String HTTP_RESPONSE_SEPARATOR = "\n"; + private static final String ARCHIVER_PATH = "archiver/"; + private static final String DATA_OUTPUT_PATH = "/data/output/"; + + /** + * Returns all the paths for every file generated by the {@link ImageTask} + * that had its ID specified. + * + * @param properties Properties that contains Object Store information. + * @param imageTaskId ImageTask's ID. + * @return List of paths. + */ + public static List getImageTaskFilesPaths(Properties properties, String imageTaskId) { + List imageTaskFilesPaths = new ArrayList<>(); + try { + HttpClient client = HttpClients.createDefault(); + HttpGet httpget = prepareObjectStoreRequest(properties, imageTaskId); + HttpResponse response = client.execute(httpget); + imageTaskFilesPaths = parseHttpResponse(response); + } catch (IOException | URISyntaxException e) { + LOGGER.error("Error while retrieving path of files of ImageTask" + + " with ID: " + imageTaskId, e); + } + return imageTaskFilesPaths; + } + + /** + * Parses the given {@link HttpResponse} to a list of Strings. + * + * @param response Response to be parsed. + * @return List of Strings parsed from response. + * @throws IOException + */ + private static List parseHttpResponse(HttpResponse response) throws IOException { + return Arrays.asList(EntityUtils.toString(response.getEntity()).split(HTTP_RESPONSE_SEPARATOR)); + } + + /** + * Prepares a {@link HttpGet} request for the Object Store. + * + * @param properties Properties that contains Object Store information. + * @param imageTaskId ImageTask's ID. + * @return A {@link HttpGet} request. + * @throws URISyntaxException + */ + private static HttpGet prepareObjectStoreRequest(Properties properties, + String imageTaskId) throws URISyntaxException { + String objectStoreHost = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_HOST); + String objectStorePath = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_PATH); + String objectStoreContainer = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_CONTAINER); + String pathParamValue = ARCHIVER_PATH + imageTaskId + DATA_OUTPUT_PATH; + URI uri = new URIBuilder() + .setScheme(HTTPS_SCHEME) + .setHost(objectStoreHost) + .setPath(objectStorePath + "/" + objectStoreContainer) + .addParameter(PATH_PARAM, pathParamValue) + .build(); + LOGGER.debug("Getting list of files for task " + imageTaskId + " from " + uri); + HttpGet httpget = new HttpGet(uri); + Token token = getKeystoneToken(properties); + httpget.addHeader(X_AUTH_TOKEN, token.getAccessId()); + return httpget; + } + + /** + * Returns a Keystone Token for the given properties. + * + * @param properties Properties that contains Object Store information. + * @return Keystone Token. + */ + private static Token getKeystoneToken(Properties properties) { + Map credentials = new HashMap<>(); + credentials.put(PROJECT_ID, properties.getProperty(SapsPropertiesConstants.SWIFT_PROJECT_ID)); + credentials.put(USER_ID, properties.getProperty(SapsPropertiesConstants.SWIFT_USER_ID)); + credentials.put(PASSWORD, properties.getProperty(SapsPropertiesConstants.SWIFT_PASSWORD)); + credentials.put(AUTH_URL, properties.getProperty(SapsPropertiesConstants.SWIFT_AUTH_URL)); + KeystoneV3IdentityPlugin keystone = new KeystoneV3IdentityPlugin(properties); + return keystone.createToken(credentials); + } + +} diff --git a/src/main/java/org/fogbowcloud/saps/engine/core/service/ProcessedImagesService.java b/src/main/java/org/fogbowcloud/saps/engine/core/service/ProcessedImagesService.java new file mode 100644 index 000000000..9ecff7732 --- /dev/null +++ b/src/main/java/org/fogbowcloud/saps/engine/core/service/ProcessedImagesService.java @@ -0,0 +1,146 @@ +package org.fogbowcloud.saps.engine.core.service; + +import org.apache.log4j.Logger; +import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFile; +import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFileList; +import org.fogbowcloud.saps.engine.scheduler.util.SapsPropertiesConstants; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.io.File; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.SignatureException; +import java.util.*; + +/** + * Service the provides operations with processed {@link ImageTask}. + */ +public class ProcessedImagesService { + + private static final Logger LOGGER = Logger.getLogger(ProcessedImagesService.class); + + private static final String TEMP_DIR_URL = "%s?temp_url_sig=%s&temp_url_expires=%s"; + private static final String UNAVAILABLE = "UNAVAILABLE"; + private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1"; + private static final String HTTPS_SCHEME = "https://"; + private static final String FILENAME_QUERY_PARAM = "&filename="; + private static final String GET = "GET"; + private static final String OBJECT_STORE_ENCRIPTION_FORMAT = "%s\n%s\n%s"; + + /** + * Generates a {@link ImageTaskFileList} object containing every file generated + * by the processing of {@link ImageTask} that was specified. + * + * @param properties Properties that contains Object Store information. + * @param imageTask ImageTask that will have its files returned. + * @return A {@link ImageTaskFileList}. + */ + public static ImageTaskFileList generateImageTaskFiles(Properties properties, ImageTask imageTask) { + List filesPaths = ObjectStoreService.getImageTaskFilesPaths(properties, imageTask.getTaskId()); + List imageTaskFiles = new ArrayList<>(); + for (String filePath: filesPaths) { + File file = new File(filePath); + ImageTaskFile imageTaskFile = new ImageTaskFile(filePath, file.getName()); + String imageTaskFileURL; + try { + imageTaskFileURL = generateTempURL(properties, imageTaskFile); + } catch (NoSuchAlgorithmException | SignatureException | InvalidKeyException e) { + LOGGER.error("Failed to generate download link for file " + filePath, e); + imageTaskFileURL = UNAVAILABLE; + } + imageTaskFile.setURL(imageTaskFileURL); + imageTaskFiles.add(imageTaskFile); + } + return new ImageTaskFileList(imageTask, imageTaskFiles); + } + + /** + * Generates a temporary access URL for given {@link ImageTaskFile}. + * + * @param properties Properties that contains Object Store information. + * @param imageTaskFile File to have its access URL generated. + * @return Temporary access URL for given {@link ImageTaskFile}. + */ + private static String generateTempURL(Properties properties, ImageTaskFile imageTaskFile) + throws NoSuchAlgorithmException, SignatureException, InvalidKeyException { + String objectStoreHost = properties + .getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_HOST); + String objectStorePath = properties + .getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_PATH); + String objectStoreContainer = properties + .getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_CONTAINER); + String objectStoreKey = properties + .getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_KEY); + String path = generateTempURLPath( + objectStorePath, + objectStoreContainer, + imageTaskFile.getPath(), + objectStoreKey); + return HTTPS_SCHEME + + objectStoreHost + + path + + FILENAME_QUERY_PARAM + + imageTaskFile.getName(); + } + + /** + * Generates the path of a temporary URL. + * + * @param swiftPath Path to Swift Object Store. + * @param container Container of Swift Object Store. + * @param filePath Path to the file that will be accessed. + * @param key Key of encryption. + * @return Path of a temporary URL. + */ + private static String generateTempURLPath( + String swiftPath, + String container, + String filePath, + String key) + throws NoSuchAlgorithmException, SignatureException, InvalidKeyException { + Formatter objectStoreFormatter = new Formatter(); + String path = swiftPath + "/" + container + "/" + filePath; + objectStoreFormatter.format(OBJECT_STORE_ENCRIPTION_FORMAT, GET, Long.MAX_VALUE, path); + String signature = calculateRFC2104HMAC(objectStoreFormatter.toString(), key); + objectStoreFormatter.close(); + objectStoreFormatter = new Formatter(); + objectStoreFormatter.format(TEMP_DIR_URL, path, signature, Long.MAX_VALUE); + String tempURLPath = objectStoreFormatter.toString(); + objectStoreFormatter.close(); + return tempURLPath; + } + + /** + * Encrypts the given data with the given key. + * + * @param data Data to be encrypted. + * @param key Key of encryption. + * @return The encrypted string. + */ + private static String calculateRFC2104HMAC(String data, String key) + throws SignatureException, NoSuchAlgorithmException, InvalidKeyException { + SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), HMAC_SHA1_ALGORITHM); + Mac mac = Mac.getInstance(HMAC_SHA1_ALGORITHM); + mac.init(signingKey); + return parseToHexString(mac.doFinal(data.getBytes())); + } + + /** + * Parses a array of bytes to a hexadecimal string. + * + * @param bytes Array of bytes to be parsed. + * @return Hexadecimal string parsed from given array of bytes. + */ + private static String parseToHexString(byte[] bytes) { + Formatter formatter = new Formatter(); + for (byte b : bytes) { + formatter.format("%02x", b); + } + String hexString = formatter.toString(); + formatter.close(); + return hexString; + } + +} diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java index 50e43c633..e2e6328bf 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/DatabaseApplication.java @@ -1,7 +1,6 @@ package org.fogbowcloud.saps.engine.scheduler.restlet; import java.io.File; -import java.io.IOException; import java.sql.SQLException; import java.text.ParseException; import java.util.Collections; @@ -95,7 +94,7 @@ public Restlet createInboundRoot() { router.attach("/images/{imgName}", ImageResource.class); router.attach("/regions/details", RegionResource.class); router.attach("/regions/search", RegionResource.class); - router.attach("/email", ProcessedImagesResource.class); + router.attach("/email", ImagesEmailResource.class); router.attach("/archivedTasks", ArchivedTasksResource.class); return router; @@ -104,6 +103,10 @@ public Restlet createInboundRoot() { public List getTasks() throws SQLException, ParseException { return submissionDispatcher.getTaskListInDB(); } + + public List getImageTasks(String[] imageTasksIds) throws SQLException { + return submissionDispatcher.getImageTasks(imageTasksIds); + } public List getTasksInState(ImageTaskState imageState) throws SQLException { return this.submissionDispatcher.getTasksInState(imageState); diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java index 5e6318471..463604f24 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ArchivedTasksResource.java @@ -1,27 +1,36 @@ package org.fogbowcloud.saps.engine.scheduler.restlet.resource; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import org.apache.log4j.Logger; import org.fogbowcloud.saps.engine.core.dispatcher.SubmissionParameters; import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFileList; +import org.fogbowcloud.saps.engine.core.service.ProcessedImagesService; +import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import org.restlet.data.Form; +import org.restlet.data.Header; import org.restlet.data.MediaType; import org.restlet.representation.Representation; import org.restlet.representation.StringRepresentation; +import org.restlet.resource.Get; import org.restlet.resource.Post; +import org.restlet.util.Series; /** - * Responsable for retrieve information about already processed - archived - tasks. + * Responsible for retrieve information about already processed {@link ImageTask}. */ public class ArchivedTasksResource extends BaseResource { private static final Logger LOGGER = Logger.getLogger(ArchivedTasksResource.class); - public ArchivedTasksResource() { - super(); - } + private static final String REQUEST_ATTR_PROCESSED_IMAGES = "images_id[]"; /** * Returns all processed tasks that matches the execution parameters passed @@ -49,4 +58,62 @@ public Representation getProcessedTasksInInterval(Representation representation) return new StringRepresentation(resObj.toString(), MediaType.APPLICATION_JSON); } + /** + * Returns the files generated by execution of each {@link ImageTask} that + * had its ID specified. + */ + @SuppressWarnings("unchecked") + @Get + public Representation getProcessedImageTasksFiles() { + Series
series = (Series
) getRequestAttributes() + .get("org.restlet.http.headers"); + + String[] imageTasksIds = series.getValuesArray(REQUEST_ATTR_PROCESSED_IMAGES, true); + LOGGER.info("Recovering files from processed ImageTasks from list of IDs: " + + Arrays.toString(imageTasksIds)); + + List imageTaskFileLists = getImageTaskFileLists(imageTasksIds); + JSONArray imageTaskFileListsJSONArray = parseToJsonArray(imageTaskFileLists); + return new StringRepresentation(imageTaskFileListsJSONArray.toString(), MediaType.APPLICATION_JSON); + } + + /** + * Gets a list of {@link ImageTaskFileList} given the IDs of {@link ImageTask}. + * + * @param imageTasksIds IDs of ImageTasks to have its ImageTaskFileList returned. + * @return List of {@link ImageTaskFileList}. + */ + private List getImageTaskFileLists(String[] imageTasksIds) { + List imageTaskFileLists = new ArrayList<>(); + try { + List imageTasks = application.getImageTasks(imageTasksIds); + imageTaskFileLists = imageTasks.stream() + .map(imageTask -> ProcessedImagesService.generateImageTaskFiles( + application.getProperties(), + imageTask)) + .collect(Collectors.toList()); + } catch (SQLException e) { + LOGGER.error("Error while getting ImageTasks from Catalogue", e); + } + return imageTaskFileLists; + } + + /** + * Parses the specified list of {@link ImageTaskFileList} to a {@link JSONArray}. + * + * @param imageTaskFileLists List of ImageTaskFileList to be parsed. + * @return List of ImageTaskFileList parsed to JSONArray. + */ + private JSONArray parseToJsonArray(List imageTaskFileLists) { + JSONArray resultJSONArray = new JSONArray(); + try { + for (ImageTaskFileList imageTaskFileList: imageTaskFileLists) { + resultJSONArray.put(imageTaskFileList.toJSON()); + } + } catch (JSONException e) { + LOGGER.error("Error while parsing list of ImageTaskFileList to JSONArray", e); + } + return resultJSONArray; + } + } \ No newline at end of file diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ProcessedImagesResource.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java similarity index 91% rename from src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ProcessedImagesResource.java rename to src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java index f424e4345..1d2e0d470 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ProcessedImagesResource.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/restlet/resource/ImagesEmailResource.java @@ -13,9 +13,9 @@ import java.util.Arrays; import java.util.Properties; -public class ProcessedImagesResource extends BaseResource { +public class ImagesEmailResource extends BaseResource { - public static final Logger LOGGER = Logger.getLogger(ProcessedImagesResource.class); + public static final Logger LOGGER = Logger.getLogger(ImagesEmailResource.class); private static final String REQUEST_ATTR_PROCESSED_IMAGES = "images_id[]"; diff --git a/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java b/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java index 2e9859d8f..480f08f06 100644 --- a/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java +++ b/src/main/java/org/fogbowcloud/saps/engine/scheduler/util/ProcessedImagesEmailBuilder.java @@ -1,56 +1,25 @@ package org.fogbowcloud.saps.engine.scheduler.util; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; import org.apache.log4j.Logger; -import org.fogbowcloud.manager.core.plugins.identity.openstack.KeystoneV3IdentityPlugin; -import org.fogbowcloud.manager.occi.model.Token; import org.fogbowcloud.saps.engine.core.model.ImageTask; +import org.fogbowcloud.saps.engine.core.pojo.ImageTaskFileList; +import org.fogbowcloud.saps.engine.core.service.ProcessedImagesService; import org.fogbowcloud.saps.engine.scheduler.restlet.DatabaseApplication; import org.fogbowcloud.saps.notifier.GoogleMail; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; import javax.mail.MessagingException; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; -import java.security.SignatureException; import java.sql.SQLException; -import java.util.*; +import java.util.List; +import java.util.Properties; public class ProcessedImagesEmailBuilder implements Runnable { private static final Logger LOGGER = Logger.getLogger(ProcessedImagesEmailBuilder.class); - private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1"; - private static final String UNAVAILABLE = "UNAVAILABLE"; - private static final String TEMP_DIR_URL = "%s?temp_url_sig=%s&temp_url_expires=%s"; - - private static final String PROJECT_ID = "projectId"; - private static final String PASSWORD = "password"; - private static final String USER_ID = "userId"; - private static final String AUTH_URL = "authUrl"; - private static final String TASK_ID = "taskId"; - private static final String REGION = "region"; - private static final String COLLECTION_TIER_NAME = "collectionTierName"; - private static final String IMAGE_DATE = "imageDate"; - private static final String NAME = "name"; - private static final String URL = "url"; - private static final String FILES = "files"; - private static final String STATUS = "status"; - private DatabaseApplication application; private Properties properties; private String userEmail; @@ -60,8 +29,7 @@ public ProcessedImagesEmailBuilder( DatabaseApplication databaseApplication, Properties properties, String userEmail, - List images - ) { + List images) { this.application = databaseApplication; this.properties = properties; this.userEmail = userEmail; @@ -78,7 +46,6 @@ public void run() { builder.append(str).append("\n"); } LOGGER.info(builder.toString()); - StringBuilder errorBuilder = new StringBuilder(); JSONArray tasklist = generateAllTasksJsons(errorBuilder); sendTaskEmail(errorBuilder, tasklist); @@ -90,45 +57,14 @@ JSONArray generateAllTasksJsons(StringBuilder errorBuilder) { for (String str: images) { try { tasklist.put(generateTaskEmailJson(properties, str)); - } catch (IOException | URISyntaxException | IllegalArgumentException e) { - LOGGER.error("Failed to fetch image from object store.", e); - errorBuilder - .append("Failed to fetch image from object store.").append("\n") - .append(ExceptionUtils.getStackTrace(e)).append("\n"); - try { - JSONObject task = new JSONObject(); - task.put(TASK_ID, str); - task.put(STATUS, UNAVAILABLE); - tasklist.put(task); - } catch (JSONException e1) { - LOGGER.error("Failed to create UNAVAILABLE task json.", e); - } } catch (SQLException e) { LOGGER.error("Failed to fetch image from database.", e); - errorBuilder - .append("Failed to fetch image from database.").append("\n") + errorBuilder.append("Failed to fetch image from database.").append("\n") .append(ExceptionUtils.getStackTrace(e)).append("\n"); - try { - JSONObject task = new JSONObject(); - task.put(TASK_ID, str); - task.put(STATUS, UNAVAILABLE); - tasklist.put(task); - } catch (JSONException e1) { - LOGGER.error("Failed to create UNAVAILABLE task json.", e); - } } catch (JSONException e) { LOGGER.error("Failed to create task json.", e); - errorBuilder - .append("Failed to create task json.").append("\n") + errorBuilder.append("Failed to create task json.").append("\n") .append(ExceptionUtils.getStackTrace(e)).append("\n"); - try { - JSONObject task = new JSONObject(); - task.put(TASK_ID, str); - task.put(STATUS, UNAVAILABLE); - tasklist.put(task); - } catch (JSONException e1) { - LOGGER.error("Failed to create UNAVAILABLE task json.", e); - } } } return tasklist; @@ -167,131 +103,12 @@ private void sendErrorEmail(StringBuilder errorBuilder) { } } - private String toHexString(byte[] bytes) { - Formatter formatter = new Formatter(); - - for (byte b : bytes) { - formatter.format("%02x", b); - } - - String hexString = formatter.toString(); - formatter.close(); - - return hexString; - } - - private String calculateRFC2104HMAC(String data, String key) - throws SignatureException, NoSuchAlgorithmException, InvalidKeyException - { - SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), HMAC_SHA1_ALGORITHM); - Mac mac = Mac.getInstance(HMAC_SHA1_ALGORITHM); - mac.init(signingKey); - return toHexString(mac.doFinal(data.getBytes())); - } - - String generateTempURL(String swiftPath, String container, String filePath, String key) - throws NoSuchAlgorithmException, SignatureException, InvalidKeyException { - String path = swiftPath + "/" + container + "/" + filePath; - - Formatter objectStoreFormatter = new Formatter(); - objectStoreFormatter.format("%s\n%s\n%s", "GET", Long.MAX_VALUE, path); - String signature = calculateRFC2104HMAC(objectStoreFormatter.toString(), key); - objectStoreFormatter.close(); - - objectStoreFormatter = new Formatter(); - objectStoreFormatter.format( - TEMP_DIR_URL, - path, - signature, - Long.MAX_VALUE - ); - String res = objectStoreFormatter.toString(); - objectStoreFormatter.close(); - - return res; - } - - JSONObject generateTaskEmailJson(Properties properties, String imageid) - throws URISyntaxException, IOException, SQLException, JSONException { - JSONObject result = new JSONObject(); - - String objectStoreHost = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_HOST); - String objectStorePath = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_PATH); - String objectStoreContainer = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_CONTAINER); - String objectStoreKey = properties.getProperty(SapsPropertiesConstants.SWIFT_OBJECT_STORE_KEY); - - ImageTask task = application.getTask(imageid); - - List files = getTaskFilesFromObjectStore( - properties, objectStoreHost, objectStorePath, objectStoreContainer, task - ); - - result.put(TASK_ID, task.getTaskId()); - result.put(REGION, task.getRegion()); - result.put(COLLECTION_TIER_NAME, task.getCollectionTierName()); - result.put(IMAGE_DATE, task.getImageDate()); - - JSONArray filelist = new JSONArray(); - for (String str: files) { - File f = new File(str); - String fileName = f.getName(); - try { - JSONObject fileobject = new JSONObject(); - fileobject.put(NAME, fileName); - fileobject.put(URL, "https://" + objectStoreHost + generateTempURL( - objectStorePath, - objectStoreContainer, - str, - objectStoreKey) + "&filename=" + fileName - ); - filelist.put(fileobject); - } catch (NoSuchAlgorithmException | SignatureException | InvalidKeyException e) { - LOGGER.error("Failed to generate download link for file " + str, e); - try { - JSONObject fileobject = new JSONObject(); - fileobject.put(NAME, fileName); - fileobject.put(URL, UNAVAILABLE); - filelist.put(fileobject); - } catch (JSONException e1) { - LOGGER.error("Failed to create UNAVAILABLE temp url json.", e); - } - } - } - result.put(FILES, filelist); - - return result; - } - - List getTaskFilesFromObjectStore(Properties properties, String objectStoreHost, String objectStorePath, String objectStoreContainer, ImageTask task) throws URISyntaxException, IOException { - Token token = getKeystoneToken(properties); - - HttpClient client = HttpClients.createDefault(); - HttpGet httpget = prepObjectStoreRequest(objectStoreHost, objectStorePath, objectStoreContainer, task, token); - HttpResponse response = client.execute(httpget); - - return Arrays.asList(EntityUtils.toString(response.getEntity()).split("\n")); - } - - private HttpGet prepObjectStoreRequest(String objectStoreHost, String objectStorePath, String objectStoreContainer, ImageTask task, Token token) throws URISyntaxException { - URI uri = new URIBuilder() - .setScheme("https") - .setHost(objectStoreHost) - .setPath(objectStorePath + "/" + objectStoreContainer) - .addParameter("path", "archiver/" + task.getTaskId() + "/data/output/") - .build(); - LOGGER.debug("Getting list of files for task " + task.getTaskId() + " from " + uri); - HttpGet httpget = new HttpGet(uri); - httpget.addHeader("X-Auth-Token", token.getAccessId()); - return httpget; + JSONObject generateTaskEmailJson(Properties properties, String imageTaskId) + throws SQLException, JSONException { + ImageTask task = application.getTask(imageTaskId); + ImageTaskFileList imageTaskFileList = ProcessedImagesService + .generateImageTaskFiles(properties, task); + return imageTaskFileList.toJSON(); } - private Token getKeystoneToken(Properties properties) { - KeystoneV3IdentityPlugin keystone = new KeystoneV3IdentityPlugin(properties); - Map credentials = new HashMap<>(); - credentials.put(AUTH_URL, properties.getProperty(SapsPropertiesConstants.SWIFT_AUTH_URL)); - credentials.put(PROJECT_ID, properties.getProperty(SapsPropertiesConstants.SWIFT_PROJECT_ID)); - credentials.put(USER_ID, properties.getProperty(SapsPropertiesConstants.SWIFT_USER_ID)); - credentials.put(PASSWORD, properties.getProperty(SapsPropertiesConstants.SWIFT_PASSWORD)); - return keystone.createToken(credentials); - } }