From 1db8926c31c2b608f5a79b826b147eaaad7906cf Mon Sep 17 00:00:00 2001 From: Alexandre Payot Date: Mon, 16 Jan 2023 14:11:17 +0000 Subject: [PATCH 1/7] Remove mount wait loop --- .gradient/prepare-datasets.sh | 36 +++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/.gradient/prepare-datasets.sh b/.gradient/prepare-datasets.sh index 3c910aa..a7f89e5 100755 --- a/.gradient/prepare-datasets.sh +++ b/.gradient/prepare-datasets.sh @@ -1,30 +1,38 @@ #!/bin/bash symlink-public-resources() { + # This function will "symlink" one read only folder and make it appear + # as read write in the target directory. This is used to make gradient + # datasets appear in directories with read/write permissions. + public_source_dir=${1} target_dir=${2} - # need to wait until the dataset has been mounted (async on Paperspace's end) - #while [ ! -d "${PUBLIC_DATASET_DIR}/exe_cache" ] - while [ ! -d ${public_source_dir} ] - do - echo "Waiting for dataset "${public_source_dir}" to be mounted..." - sleep 1 - done - + MAX_MOUNT_TIME=60 + WAITING_FOR=0 + # while [ ! -d ${public_source_dir} ] && [ "$((${WAITING_FOR}<${MAX_MOUNT_TIME}))" -eq 1 ] + # do + # echo "Waiting for dataset "${public_source_dir}" to be mounted..." + # sleep 1 + # done + # To use an overlay mount in a container we need to make sure that the + # work and upper directories are not themselves in overlays. + OVERLAY_DIRECTORY="/fusedoverlay" + if [ ! -d ${OVERLAY_DIRECTORY} ]; then + echo "Mounting new tmpfs to ${OVERLAY_DIRECTORY}" + mount -t tmpfs tmpfs ${OVERLAY_DIRECTORY} + fi echo "Symlinking - ${public_source_dir} to ${target_dir}" - # Make sure it exists otherwise you'll copy your current dir mkdir -p ${target_dir} - workdir="/fusedoverlay/workdirs/${public_source_dir}" - upperdir="/fusedoverlay/upperdir/${public_source_dir}" + workdir="${OVERLAY_DIRECTORY}/workdirs/${public_source_dir}" + upperdir="${OVERLAY_DIRECTORY}/upperdir/${public_source_dir}" mkdir -p ${workdir} mkdir -p ${upperdir} - fuse-overlayfs -o lowerdir=${public_source_dir},upperdir=${upperdir},workdir=${workdir} ${target_dir} + mount -t overlay overlay -o lowerdir=${public_source_dir},upperdir=${upperdir},workdir=${workdir} ${target_dir} } -apt update -y -apt install -y libfuse3-dev fuse-overlayfs + echo "Starting preparation of datasets" # symlink exe_cache files From 653f44c8f5835d518858c1e06c0a8474d3774fbc Mon Sep 17 00:00:00 2001 From: Alexandre Payot Date: Mon, 16 Jan 2023 14:15:16 +0000 Subject: [PATCH 2/7] Create all directories --- .gradient/prepare-datasets.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gradient/prepare-datasets.sh b/.gradient/prepare-datasets.sh index a7f89e5..226a90b 100755 --- a/.gradient/prepare-datasets.sh +++ b/.gradient/prepare-datasets.sh @@ -20,10 +20,12 @@ symlink-public-resources() { OVERLAY_DIRECTORY="/fusedoverlay" if [ ! -d ${OVERLAY_DIRECTORY} ]; then echo "Mounting new tmpfs to ${OVERLAY_DIRECTORY}" + mkdir -p /fusedoverlay mount -t tmpfs tmpfs ${OVERLAY_DIRECTORY} fi echo "Symlinking - ${public_source_dir} to ${target_dir}" + mkdir -p ${public_source_dir} mkdir -p ${target_dir} workdir="${OVERLAY_DIRECTORY}/workdirs/${public_source_dir}" upperdir="${OVERLAY_DIRECTORY}/upperdir/${public_source_dir}" From ac150f26b1b6ace294b2e2ffb862cd54dcb77c9d Mon Sep 17 00:00:00 2001 From: Alexandre Payot Date: Mon, 16 Jan 2023 14:20:15 +0000 Subject: [PATCH 3/7] Add back the check for mounting the dataset --- .gradient/prepare-datasets.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.gradient/prepare-datasets.sh b/.gradient/prepare-datasets.sh index 226a90b..20a13dc 100755 --- a/.gradient/prepare-datasets.sh +++ b/.gradient/prepare-datasets.sh @@ -8,13 +8,20 @@ symlink-public-resources() { public_source_dir=${1} target_dir=${2} # need to wait until the dataset has been mounted (async on Paperspace's end) + # otherwise the mount will not be successful MAX_MOUNT_TIME=60 WAITING_FOR=0 - # while [ ! -d ${public_source_dir} ] && [ "$((${WAITING_FOR}<${MAX_MOUNT_TIME}))" -eq 1 ] - # do - # echo "Waiting for dataset "${public_source_dir}" to be mounted..." - # sleep 1 - # done + while [ ! -d ${public_source_dir} ] && [ "$((${WAITING_FOR}<${MAX_MOUNT_TIME}))" -eq 1 ] + do + echo "Waiting for dataset "${public_source_dir}" to be mounted..." + WAITING_FOR=$((${WAITING_FOR} + 1)) + sleep 1 + done + if [ ! -d ${public_source_dir} ] + then + echo "Error: Cannot symlink ${public_source_dir} it was not mounted in ${MAX_MOUNT_TIME}s" + return -1 + fi # To use an overlay mount in a container we need to make sure that the # work and upper directories are not themselves in overlays. OVERLAY_DIRECTORY="/fusedoverlay" @@ -25,7 +32,6 @@ symlink-public-resources() { fi echo "Symlinking - ${public_source_dir} to ${target_dir}" - mkdir -p ${public_source_dir} mkdir -p ${target_dir} workdir="${OVERLAY_DIRECTORY}/workdirs/${public_source_dir}" upperdir="${OVERLAY_DIRECTORY}/upperdir/${public_source_dir}" From e1404a15c09515d697fd3e53c8090852460a6d76 Mon Sep 17 00:00:00 2001 From: Alexandre Payot Date: Mon, 16 Jan 2023 17:16:23 +0000 Subject: [PATCH 4/7] Use variable everywhere we can --- .gradient/prepare-datasets.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gradient/prepare-datasets.sh b/.gradient/prepare-datasets.sh index 20a13dc..926f442 100755 --- a/.gradient/prepare-datasets.sh +++ b/.gradient/prepare-datasets.sh @@ -27,7 +27,7 @@ symlink-public-resources() { OVERLAY_DIRECTORY="/fusedoverlay" if [ ! -d ${OVERLAY_DIRECTORY} ]; then echo "Mounting new tmpfs to ${OVERLAY_DIRECTORY}" - mkdir -p /fusedoverlay + mkdir -p ${OVERLAY_DIRECTORY} mount -t tmpfs tmpfs ${OVERLAY_DIRECTORY} fi echo "Symlinking - ${public_source_dir} to ${target_dir}" From 076ebe456bfd8f119e629e6447f85da962f49a18 Mon Sep 17 00:00:00 2001 From: Alexandre Payot Date: Mon, 16 Jan 2023 17:32:35 +0000 Subject: [PATCH 5/7] Chmod files in the overlay directory --- .gradient/prepare-datasets.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gradient/prepare-datasets.sh b/.gradient/prepare-datasets.sh index 926f442..39c328d 100755 --- a/.gradient/prepare-datasets.sh +++ b/.gradient/prepare-datasets.sh @@ -38,7 +38,8 @@ symlink-public-resources() { mkdir -p ${workdir} mkdir -p ${upperdir} mount -t overlay overlay -o lowerdir=${public_source_dir},upperdir=${upperdir},workdir=${workdir} ${target_dir} - + find ${target_dir} -type d -print0 | xargs -0 chmod 777 + find ${target_dir} -type f -print0 | xargs -0 chmod 666 } From a8ff1e64502f9bc850c0cee3088c309d4f840f2d Mon Sep 17 00:00:00 2001 From: Alexandre Payot <18074599+payoto@users.noreply.github.com> Date: Thu, 26 Jan 2023 15:46:14 +0000 Subject: [PATCH 6/7] Move the temp directory into tmp --- .gradient/prepare-datasets.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gradient/prepare-datasets.sh b/.gradient/prepare-datasets.sh index 39c328d..cf8369a 100755 --- a/.gradient/prepare-datasets.sh +++ b/.gradient/prepare-datasets.sh @@ -24,7 +24,7 @@ symlink-public-resources() { fi # To use an overlay mount in a container we need to make sure that the # work and upper directories are not themselves in overlays. - OVERLAY_DIRECTORY="/fusedoverlay" + OVERLAY_DIRECTORY="/tmp/fusedoverlay" if [ ! -d ${OVERLAY_DIRECTORY} ]; then echo "Mounting new tmpfs to ${OVERLAY_DIRECTORY}" mkdir -p ${OVERLAY_DIRECTORY} From d01b48d4026fb9824087747bd5751b0e849f5c75 Mon Sep 17 00:00:00 2001 From: Alexandre Payot <18074599+payoto@users.noreply.github.com> Date: Thu, 26 Jan 2023 16:06:17 +0000 Subject: [PATCH 7/7] Remove workdir and upperdir double slashes --- .gradient/prepare-datasets.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gradient/prepare-datasets.sh b/.gradient/prepare-datasets.sh index cf8369a..f28491a 100755 --- a/.gradient/prepare-datasets.sh +++ b/.gradient/prepare-datasets.sh @@ -33,8 +33,8 @@ symlink-public-resources() { echo "Symlinking - ${public_source_dir} to ${target_dir}" mkdir -p ${target_dir} - workdir="${OVERLAY_DIRECTORY}/workdirs/${public_source_dir}" - upperdir="${OVERLAY_DIRECTORY}/upperdir/${public_source_dir}" + workdir="${OVERLAY_DIRECTORY}/workdirs${public_source_dir}" + upperdir="${OVERLAY_DIRECTORY}/upperdir${public_source_dir}" mkdir -p ${workdir} mkdir -p ${upperdir} mount -t overlay overlay -o lowerdir=${public_source_dir},upperdir=${upperdir},workdir=${workdir} ${target_dir}