Skip to content

Function for generic export #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yfan2012 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Function for generic export #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
90d2db5
add move_files to help with generic export
yfan2012 Apr 12, 2024
2f82d92
unify function definition and add to base
yfan2012 Apr 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions src/cwl_platform/arvados_platform.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,83 @@ def get_project_by_id(self, project_id):
return search_result['items'][0]
return None

def move_files(self, project, files, destination):
'''
take list of file paths
copy to destination
if the destination is already the prefix to any filepaths, it will not nest the destination path

:param project: The project to stage files to
:param files: list of filepaths that need to be moved
:param destination: destination folder.
'''

# set up destination collection name and path
if destination.startswith('/'):
dest_name = destination.split('/')[1]
dest_path = '/'.join(destination.split('/')[2:]) + '/'
else:
dest_name = destination.split('/')[0]
dest_path = '/'.join(destination.split('/')[1:]) + '/'

# Get the destination project collection
search_result = self.api.collections().list(filters=[
["owner_uuid", "=", project["uuid"]],
["name", "=", dest_name]
]).execute()
if len(search_result['items']) > 0:
dest_coll = search_result['items'][0]
dest_coll_filesgen = arvados.collection.CollectionReader(dest_coll['current_version_uuid']).all_files()
dest_coll_files = [j.stream_name()[2:] + '/' + j.name() for j in dest_coll_filesgen]
else:
dest_coll = self.api.collections().create(body={
"owner_uuid": project["uuid"],
"name": dest_name,
"preserve_version":True}).execute()
dest_coll_files = []

# Get target
target = arvados.collection.Collection(dest_coll['uuid'], api_client=self.api, keep_client=self.keep_client)

# loop through file paths
for i in files:
if i.startswith('/'):
orig_name = i.split('/')[1]
orig_path = '/'.join(i.split('/')[2:])
else:
orig_name = i.split('/')[0]
orig_path = '/'.join(i.split('/')[1:])
try:
# check if file exists and prepend new version number if so
count = 0
dup = True
if dest_path.startswith('/'):
clean_dest = dest_path[1:]
else:
clean_dest = dest_path
new_path = clean_dest + orig_path

while dup:
if new_path in dest_coll_files:
count += 1
new_path = clean_dest[:-1] + '/'.join(orig_path.split('/')[:-1]) + '/v' + str(count) + '_' + orig_path.split('/')[-1]
else:
dup = False

search_result = self.api.collections().list(filters=[
["owner_uuid", "=", project["uuid"]],
["name", "=", orig_name]
]).execute()
orig_coll = search_result['items'][0]
source = arvados.collection.CollectionReader(orig_coll['uuid'])
target.copy(orig_path,
target_path = new_path,
source_collection = source,
overwrite = False)
except IOError:
continue
target.save()

def stage_output_files(self, project, output_files):
'''
Stage output files to a project
Expand Down
12 changes: 12 additions & 0 deletions src/cwl_platform/base_platform.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,18 @@ def set_logger(self, logger):
''' Set the logger '''
self.logger = logger

@abstractmethod
def move_files(self, project, files, destination):
'''
take list of file paths
copy to destination
if the destination is already the prefix to any filepaths, it will not nest the destination path

:param project: The project to stage files to
:param files: list of filepaths that need to be moved
:param destination: destination folder.
'''

@abstractmethod
def stage_output_files(self, project, output_files):
'''
Expand Down
26 changes: 26 additions & 0 deletions src/cwl_platform/sevenbridges_platform.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,32 @@ def get_project_by_id(self, project_id):
''' Get a project by its id '''
return self.api.projects.get(project_id)

def move_files(self, project, files, destination):
'''
take list of file paths
copy to destination
if the destination is already the prefix to any filepaths, it will not nest the destination path

:param project: The project to stage files to
:param files: list of filepaths that need to be moved
:param destination: destination folder.
'''
for i in files:
logger.info('moving this file ' + i)
fileid = self.get_file_id(project, i)
f = self.api.files.get(fileid)

# check if destination already matches
if i.startswith(destination):
dest = self._find_or_create_path(project, '/' + '/'.join(i.split('/')[0:-1]))
else:
if len(i.split('/')) > 1:
dest = self._find_or_create_path(project, destination + '/' + '/'.join(i.split('/')[0:-1]))
else:
dest = self._find_or_create_path(project, destination)
f.move_to_folder(parent = dest, name = i.split('/')[-1])
return destination

def stage_output_files(self, project, output_files):
'''
Stage output files to a project
Expand Down