diff --git a/src/cwl_platform/arvados_platform.py b/src/cwl_platform/arvados_platform.py index cb13cb2c..7270eaa4 100644 --- a/src/cwl_platform/arvados_platform.py +++ b/src/cwl_platform/arvados_platform.py @@ -436,6 +436,83 @@ def get_project_by_id(self, project_id): return search_result['items'][0] return None + def move_files(self, project, files, destination): + ''' + take list of file paths + copy to destination + if the destination is already the prefix to any filepaths, it will not nest the destination path + + :param project: The project to stage files to + :param files: list of filepaths that need to be moved + :param destination: destination folder. + ''' + + # set up destination collection name and path + if destination.startswith('/'): + dest_name = destination.split('/')[1] + dest_path = '/'.join(destination.split('/')[2:]) + '/' + else: + dest_name = destination.split('/')[0] + dest_path = '/'.join(destination.split('/')[1:]) + '/' + + # Get the destination project collection + search_result = self.api.collections().list(filters=[ + ["owner_uuid", "=", project["uuid"]], + ["name", "=", dest_name] + ]).execute() + if len(search_result['items']) > 0: + dest_coll = search_result['items'][0] + dest_coll_filesgen = arvados.collection.CollectionReader(dest_coll['current_version_uuid']).all_files() + dest_coll_files = [j.stream_name()[2:] + '/' + j.name() for j in dest_coll_filesgen] + else: + dest_coll = self.api.collections().create(body={ + "owner_uuid": project["uuid"], + "name": dest_name, + "preserve_version":True}).execute() + dest_coll_files = [] + + # Get target + target = arvados.collection.Collection(dest_coll['uuid'], api_client=self.api, keep_client=self.keep_client) + + # loop through file paths + for i in files: + if i.startswith('/'): + orig_name = i.split('/')[1] + orig_path = '/'.join(i.split('/')[2:]) + else: + orig_name = i.split('/')[0] + orig_path = '/'.join(i.split('/')[1:]) + try: + # check if file exists and prepend new version number if so + count = 0 + dup = True + if dest_path.startswith('/'): + clean_dest = dest_path[1:] + else: + clean_dest = dest_path + new_path = clean_dest + orig_path + + while dup: + if new_path in dest_coll_files: + count += 1 + new_path = clean_dest[:-1] + '/'.join(orig_path.split('/')[:-1]) + '/v' + str(count) + '_' + orig_path.split('/')[-1] + else: + dup = False + + search_result = self.api.collections().list(filters=[ + ["owner_uuid", "=", project["uuid"]], + ["name", "=", orig_name] + ]).execute() + orig_coll = search_result['items'][0] + source = arvados.collection.CollectionReader(orig_coll['uuid']) + target.copy(orig_path, + target_path = new_path, + source_collection = source, + overwrite = False) + except IOError: + continue + target.save() + def stage_output_files(self, project, output_files): ''' Stage output files to a project diff --git a/src/cwl_platform/base_platform.py b/src/cwl_platform/base_platform.py index 17a7b1df..f39a15ce 100644 --- a/src/cwl_platform/base_platform.py +++ b/src/cwl_platform/base_platform.py @@ -102,6 +102,18 @@ def set_logger(self, logger): ''' Set the logger ''' self.logger = logger + @abstractmethod + def move_files(self, project, files, destination): + ''' + take list of file paths + copy to destination + if the destination is already the prefix to any filepaths, it will not nest the destination path + + :param project: The project to stage files to + :param files: list of filepaths that need to be moved + :param destination: destination folder. + ''' + @abstractmethod def stage_output_files(self, project, output_files): ''' diff --git a/src/cwl_platform/sevenbridges_platform.py b/src/cwl_platform/sevenbridges_platform.py index e6f8f176..df337519 100644 --- a/src/cwl_platform/sevenbridges_platform.py +++ b/src/cwl_platform/sevenbridges_platform.py @@ -433,6 +433,32 @@ def get_project_by_id(self, project_id): ''' Get a project by its id ''' return self.api.projects.get(project_id) + def move_files(self, project, files, destination): + ''' + take list of file paths + copy to destination + if the destination is already the prefix to any filepaths, it will not nest the destination path + + :param project: The project to stage files to + :param files: list of filepaths that need to be moved + :param destination: destination folder. + ''' + for i in files: + logger.info('moving this file ' + i) + fileid = self.get_file_id(project, i) + f = self.api.files.get(fileid) + + # check if destination already matches + if i.startswith(destination): + dest = self._find_or_create_path(project, '/' + '/'.join(i.split('/')[0:-1])) + else: + if len(i.split('/')) > 1: + dest = self._find_or_create_path(project, destination + '/' + '/'.join(i.split('/')[0:-1])) + else: + dest = self._find_or_create_path(project, destination) + f.move_to_folder(parent = dest, name = i.split('/')[-1]) + return destination + def stage_output_files(self, project, output_files): ''' Stage output files to a project