From 5a6ad2fb3fbc56e36cfe9832f5e319f45bf39761 Mon Sep 17 00:00:00 2001 From: Jonathan Steel Date: Tue, 14 Jan 2020 15:41:11 +0000 Subject: [PATCH] Test scripts To test the functionality of the DAC --- tools/slurm-test.sh | 121 +------------------------------ tools/tests/config | 6 ++ tools/tests/functions | 14 ++++ tools/tests/over_allocation.sh | 42 +++++++++++ tools/tests/perjob_buffer.sh | 102 ++++++++++++++++++++++++++ tools/tests/persistent_buffer.sh | 55 ++++++++++++++ tools/tests/template | 25 +++++++ 7 files changed, 248 insertions(+), 117 deletions(-) create mode 100644 tools/tests/config create mode 100644 tools/tests/functions create mode 100755 tools/tests/over_allocation.sh create mode 100755 tools/tests/perjob_buffer.sh create mode 100755 tools/tests/persistent_buffer.sh create mode 100644 tools/tests/template diff --git a/tools/slurm-test.sh b/tools/slurm-test.sh index b8b9f7e9..9d0fe7df 100755 --- a/tools/slurm-test.sh +++ b/tools/slurm-test.sh @@ -1,120 +1,7 @@ #!/bin/bash -set -eux -cd /data -echo "#!/bin/bash -#BB create_persistent name=mytestbuffer capacity=4000GB access=striped type=scratch" > create-persistent.sh +source "$( dirname $0 )/tests/config" -echo "#!/bin/bash -#BB destroy_persistent name=mytestbuffer" > delete-persistent.sh - -echo "#!/bin/bash -#DW jobdw capacity=2TB access_mode=striped,private type=scratch -#DW stage_in source=/usr/local/bin/dacd destination=\$DW_JOB_STRIPED/filename1 type=file -#DW stage_out source=\$DW_JOB_STRIPED/outdir destination=/tmp/perjob type=directory - -env -df -h - -mkdir \$DW_JOB_STRIPED/outdir -df -h > \$DW_JOB_STRIPED/outdir/dfoutput -ls -al \$DW_JOB_STRIPED > \$DW_JOB_STRIPED/outdir/lsoutput -file \$DW_JOB_STRIPED/filename1 > \$DW_JOB_STRIPED/outdir/stageinfile - -echo \$HOSTNAME -" > use-perjob.sh - -echo "#!/bin/bash -#DW persistentdw name=mytestbuffer -#DW stage_in source=/usr/local/bin/dacd destination=\$DW_PERSISTENT_STRIPED_mytestbuffer/filename1 type=file -#DW stage_out source=\$DW_PERSISTENT_STRIPED_mytestbuffer/outdir destination=/tmp/persistent type=directory - -env -df -h - -mkdir -p \$DW_PERSISTENT_STRIPED_mytestbuffer/outdir -echo \$SLURM_JOBID >> \$DW_PERSISTENT_STRIPED_mytestbuffer/outdir/jobids -ls -al \$DW_PERSISTENT_STRIPED_mytestbuffer >> \$DW_PERSISTENT_STRIPED_mytestbuffer/outdir/lsoutput -file \$DW_PERSISTENT_STRIPED_mytestbuffer/filename1 >> \$DW_PERSISTENT_STRIPED_mytestbuffer/outdir/stageinfile - -echo \$HOSTNAME -" > use-persistent.sh - -echo "#!/bin/bash -#DW jobdw capacity=2TB access_mode=striped,private type=scratch -#DW persistentdw name=mytestbuffer -#DW stage_in source=/usr/local/bin/dacd destination=\$DW_PERSISTENT_STRIPED_mytestbuffer/filename1 type=file -#DW stage_out source=\$DW_PERSISTENT_STRIPED_mytestbuffer/outdir destination=/tmp/persistent type=directory - -env -df -h - -touch \$DW_JOB_STRIPED/\$( date +%F ) -ls -al \$DW_JOB_STRIPED - -mkdir -p \$DW_PERSISTENT_STRIPED_mytestbuffer/outdir -echo \$SLURM_JOBID >> \$DW_PERSISTENT_STRIPED_mytestbuffer/outdir/jobids -ls -al \$DW_PERSISTENT_STRIPED_mytestbuffer >> \$DW_PERSISTENT_STRIPED_mytestbuffer/outdir/lsoutput -file \$DW_PERSISTENT_STRIPED_mytestbuffer/filename1 >> \$DW_PERSISTENT_STRIPED_mytestbuffer/outdir/stageinfile - -echo \$HOSTNAME -" > use-multiple.sh - -# Ensure Slurm is setup with the cluster name -/usr/bin/sacctmgr --immediate add cluster name=linux || true - -SLEEP_INTERVAL=10 - -echo "Wait for startup to complete..." -sleep $SLEEP_INTERVAL - -scontrol show burstbuffer -scontrol show bbstat - -echo "***Create persistent buffer***" -cat create-persistent.sh -su slurm -c 'sbatch create-persistent.sh' - -sleep $SLEEP_INTERVAL -scontrol show burstbuffer -squeue - -echo "***Create per job buffer***" -su slurm -c 'srun --bb="capacity=100GB" bash -c "sleep 5 && echo \$HOSTNAME"' - -scontrol show burstbuffer -squeue - -echo "***Use persistent buffer***" -id centos &>/dev/null || adduser centos -cat use-multiple.sh -su centos -c 'sbatch --array=1-10 use-multiple.sh' -su centos -c 'sbatch use-multiple.sh' -su centos -c 'sbatch use-multiple.sh' -su centos -c 'sbatch use-multiple.sh' -su centos -c 'sbatch use-multiple.sh' -cat use-persistent.sh -su centos -c 'sbatch use-persistent.sh' -cat use-perjob.sh -su centos -c 'sbatch use-perjob.sh' - -squeue - -sleep $SLEEP_INTERVAL -scontrol show burstbuffer -squeue -sleep $SLEEP_INTERVAL - -#echo "***Delete persistent buffer***" -#cat delete-persistent.sh -#su slurm -c 'sbatch delete-persistent.sh' -#sleep $SLEEP_INTERVAL - -#echo "***Show all buffers are cleaned up***" -scontrol show burstbuffer -squeue -sleep $SLEEP_INTERVAL -scontrol show burstbuffer -squeue -scontrol show bbstat -sacct -o jobid'%10',reqtres'%45',alloctres'%45' -X +for i in tests/*.sh; do + timeout $timeout $i || echo test $i timed out +done diff --git a/tools/tests/config b/tools/tests/config new file mode 100644 index 00000000..57bc5a51 --- /dev/null +++ b/tools/tests/config @@ -0,0 +1,6 @@ +file=bufferfile.txt +outfile=bufferout.txt +timeout=1h +jobtimeout=30m +size=2TB +nodes=2 diff --git a/tools/tests/functions b/tools/tests/functions new file mode 100644 index 00000000..13c52c88 --- /dev/null +++ b/tools/tests/functions @@ -0,0 +1,14 @@ +wait_for_jobs() { + set +e + timeout $jobtimeout bash -c \ + 'until [ -z "$(squeue -h -n $name)" ]; do + sleep 10 + done' + timeoutrc=$? + set -e + if [ ! $timeoutrc -eq 0 ]; then + echo timeout $jobtimeout reached waiting for jobs to complete: + squeue -n $name + return 1 + fi +} diff --git a/tools/tests/over_allocation.sh b/tools/tests/over_allocation.sh new file mode 100755 index 00000000..ff695f13 --- /dev/null +++ b/tools/tests/over_allocation.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -e + +export name=test$RANDOM +scratch=$HOME/dac-tests-$name +output=$scratch/slurm.out + +source "$( dirname $0 )/config" +source "$( dirname $0 )/functions" + +# Calculate the size of the buffers to use, we'll create two each needing 2/3 of the space +export $( scontrol show bu | head -n1 ) +size=$( expr ${TotalSpace//[!0-9]/} / 3 \* 2 )${TotalSpace//[0-9]/} + +create_files() { + mkdir -p $scratch + touch $scratch/$file + + echo "#!/bin/bash +#DW jobdw capacity=$size access_mode=striped type=scratch +#DW stage_in source=$scratch/$file destination=\$DW_JOB_STRIPED/$file type=file +#DW stage_out source=\$DW_JOB_STRIPED/$file destination=$scratch/$outfile type=file +#SBATCH --output=$output --open-mode=append +test -n \"\$DW_JOB_STRIPED\" || echo 'environment variable \$DW_JOB_STRIPED is not set' +test -d \"\$DW_JOB_STRIPED\" || echo 'environment variable \$DW_JOB_STRIPED is not a directory' +touch \"\$DW_JOB_STRIPED/$file\" || echo 'failed to touch file in \$DW_JOB_STRIPED'" > $scratch/over-allocate.sh +} + +submit_jobs() { + for num in {1..2}; do + sbatch --job-name=$name --quiet $scratch/over-allocate.sh + done +} + +cleanup() { + scancel -n $name -u $USER + rm -rf "$scratch" + wait_for_jobs +} + +source "$( dirname $0 )/template" diff --git a/tools/tests/perjob_buffer.sh b/tools/tests/perjob_buffer.sh new file mode 100755 index 00000000..bca63a77 --- /dev/null +++ b/tools/tests/perjob_buffer.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +set -e + +export name=test$RANDOM +scratch=$HOME/dac-tests-$name +output=$scratch/slurm.out + +source "$( dirname $0 )/config" +source "$( dirname $0 )/functions" + +create_files() { + mkdir -p $scratch + touch $scratch/$file + + echo "#!/bin/bash +#DW jobdw capacity=$size access_mode=private type=scratch +#SBATCH --output=$output --open-mode=append --nodes=$nodes +set -e +test -n \"\$DW_JOB_PRIVATE\" || echo 'environment variable \$DW_JOB_PRIVATE is not set' +test -d \"\$DW_JOB_PRIVATE\" || echo 'environment variable \$DW_JOB_PRIVATE is not a directory' +touch \"\$DW_JOB_PRIVATE/$file\" || echo 'failed to touch file in \$DW_JOB_PRIVATE' +stat_uid=\$( stat -c %U \$DW_JOB_PRIVATE/ ) +stat_gid=\$( stat -c %G \$DW_JOB_PRIVATE/ ) +stat_perm=\$( stat -c %a \$DW_JOB_PRIVATE/ ) +if [[ \$stat_uid != $USER ]]; then + echo \"UID of \$DW_JOB_PRIVATE is \$stat_uid and not $USER\"; fi +if [[ \$stat_gid != $USER ]]; then + echo \"GID of \$DW_JOB_PRIVATE is \$stat_gid and not $USER\"; fi +if [[ \$stat_perm != 700 ]]; then + echo \"Permissions of \$DW_JOB_PRIVATE are \$stat_perm and not 700\"; fi" > $scratch/use-private.sh + + + echo "#!/bin/bash +#DW jobdw capacity=$size access_mode=striped type=scratch +#DW stage_in source=$scratch/$file destination=\$DW_JOB_STRIPED/$file type=file +#DW stage_out source=\$DW_JOB_STRIPED/$file destination=$scratch/$outfile type=file +#SBATCH --output=$output --open-mode=append --nodes=$nodes +set -e +test -n \"\$DW_JOB_STRIPED\" || echo 'environment variable \$DW_JOB_STRIPED is not set' +test -d \"\$DW_JOB_STRIPED\" || echo 'environment variable \$DW_JOB_STRIPED is not a directory' +test -f \"\$DW_JOB_STRIPED/$file\" || echo 'stage in file \$DW_JOB_STRIPED/$file does not exist' +touch \"\$DW_JOB_STRIPED/$file\" || echo 'failed to touch file in \$DW_JOB_STRIPED' +stat_uid=\$( stat -c %U \$DW_JOB_STRIPED/ ) +stat_gid=\$( stat -c %G \$DW_JOB_STRIPED/ ) +stat_perm=\$( stat -c %a \$DW_JOB_STRIPED/ ) +if [[ \$stat_uid != $USER ]]; then + echo \"UID of \$DW_JOB_STRIPED is \$stat_uid and not $USER\"; fi +if [[ \$stat_gid != $USER ]]; then + echo \"GID of \$DW_JOB_STRIPED is \$stat_gid and not $USER\"; fi +if [[ \$stat_perm != 700 ]]; then + echo \"Permissions of \$DW_JOB_STRIPED are \$stat_perm and not 700\"; fi" > $scratch/use-striped.sh + + echo "#!/bin/bash +#DW jobdw capacity=$size access_mode=private,striped type=scratch +#DW stage_in source=$scratch/$file destination=\$DW_JOB_STRIPED/$file type=file +#DW stage_out source=\$DW_JOB_STRIPED/$file destination=$scratch/$file type=file +#SBATCH --output=$output --open-mode=append --nodes=$nodes +set -e +test -n \"\$DW_JOB_PRIVATE\" || echo 'environment variable \$DW_JOB_PRIVATE is not set' +test -n \"\$DW_JOB_STRIPED\" || echo 'environment variable \$DW_JOB_STRIPED is not set' +test -d \"\$DW_JOB_PRIVATE\" || echo 'environment variable \$DW_JOB_PRIVATE is not a directory' +test -d \"\$DW_JOB_STRIPED\" || echo 'environment variable \$DW_JOB_STRIPED is not a directory' +test -f \"\$DW_JOB_STRIPED/$file\" || echo 'stage in file \$DW_JOB_STRIPED/$file does not exist' +touch \"\$DW_JOB_PRIVATE/$file\" || echo 'failed to touch file in \$DW_JOB_PRIVATE' +touch \"\$DW_JOB_STRIPED/$file\" || echo 'failed to touch file in \$DW_JOB_STRIPED' +stat_uid=\$( stat -c %U \$DW_JOB_STRIPED/ ) +stat_gid=\$( stat -c %G \$DW_JOB_STRIPED/ ) +stat_perm=\$( stat -c %a \$DW_JOB_STRIPED/ ) +if [[ \$stat_uid != $USER ]]; then + echo \"UID of \$DW_JOB_STRIPED is \$stat_uid and not $USER\"; fi +if [[ \$stat_gid != $USER ]]; then + echo \"GID of \$DW_JOB_STRIPED is \$stat_gid and not $USER\"; fi +if [[ \$stat_perm != 700 ]]; then + echo \"Permissions of \$DW_JOB_STRIPED are \$stat_perm and not 700\"; fi +stat_uid=\$( stat -c %U \$DW_JOB_PRIVATE/ ) +stat_gid=\$( stat -c %G \$DW_JOB_PRIVATE/ ) +stat_perm=\$( stat -c %a \$DW_JOB_PRIVATE/ ) +if [[ \$stat_uid != $USER ]]; then + echo \"UID of \$DW_JOB_PRIVATE is \$stat_uid and not $USER\"; fi +if [[ \$stat_gid != $USER ]]; then + echo \"GID of \$DW_JOB_PRIVATE is \$stat_gid and not $USER\"; fi +if [[ \$stat_perm != 700 ]]; then + echo \"Permissions of \$DW_JOB_PRIVATE are \$stat_perm and not 700\"; fi" > $scratch/use-both.sh +} + +submit_jobs() { + sbatch --job-name=$name --quiet $scratch/use-striped.sh + sbatch --job-name=$name --quiet $scratch/use-private.sh + sbatch --job-name=$name --quiet $scratch/use-both.sh + sbatch --job-name=$name --quiet --array=1-10 $scratch/use-striped.sh + sbatch --job-name=$name --quiet --array=1-10 $scratch/use-private.sh + sbatch --job-name=$name --quiet --array=1-10 $scratch/use-both.sh +} + +cleanup() { + scancel -n $name -u $USER + rm -rf "$scratch" + wait_for_jobs +} + +source "$( dirname $0 )/template" diff --git a/tools/tests/persistent_buffer.sh b/tools/tests/persistent_buffer.sh new file mode 100755 index 00000000..62be9d07 --- /dev/null +++ b/tools/tests/persistent_buffer.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -e + +export name=test$RANDOM +scratch=$HOME/dac-tests-$name +output=$scratch/slurm.out + +source "$( dirname $0 )/config" +source "$( dirname $0 )/functions" + +create_files() { + mkdir -p $scratch + touch $scratch/$file + + echo "#!/bin/bash +#BB create_persistent name=$name capacity=$size access=striped type=scratch +#SBATCH --output=$output --open-mode=append +" > $scratch/create-persistent.sh + + echo "#!/bin/bash +#DW persistentdw name=$name +#DW stage_in source=$scratch/$file destination=\$DW_PERSISTENT_STRIPED_$name/$file type=file +#DW stage_out source=\$DW_PERSISTENT_STRIPED_$name/$file destination=$scratch/$outfile type=file +#SBATCH --output=$output --open-mode=append --nodes=$nodes +test -n \"\$DW_PERSISTENT_STRIPED_$name\" || echo environment variable is not set +test -d \"\$DW_PERSISTENT_STRIPED_$name\" || echo environment varibale is not a directory +test -f \"\$DW_PERSISTENT_STRIPED_$name/$file\" || echo stage in file $DW_PERSISTENT_STRIPED_$name/$file does not exist +echo \$SLURM_JOBID > \"\$DW_PERSISTENT_STRIPED_$name/$outfile\" || echo failed to write to file in buffer +if [[ \$( stat -c %U \$DW_PERSISTENT_STRIPED_$name/ ) != $USER ]]; then + echo \"UID of \$DW_PERSISTENT_STRIPED_$name is not $USER\"; fi +if [[ \$( stat -c %G \$DW_PERSISTENT_STRIPED_$name/ ) != root ]]; then + echo \"GID of \$DW_PERSISTENT_STRIPED_$name is not root\"; fi +if [[ \$( stat -c %a \$DW_PERSISTENT_STRIPED_$name/ ) != 700 ]]; then + echo \"Permissions of \$DW_PERSISTENT_STRIPED_$name is not 700\"; fi" > $scratch/use-persistent.sh + + echo "#!/bin/bash +#BB destroy_persistent name=$name +#SBATCH --output=$output --open-mode=append +" > $scratch/delete-persistent.sh +} + +submit_jobs() { + sbatch --job-name=$name --quiet $scratch/create-persistent.sh + sbatch --job-name=$name --quiet $scratch/use-persistent.sh +} + +cleanup() { + scancel -n $name -u $USER + sbatch --job-name=$name --quiet $scratch/delete-persistent.sh + rm -rf "$scratch" + wait_for_jobs +} + +source "$( dirname $0 )/template" diff --git a/tools/tests/template b/tools/tests/template new file mode 100644 index 00000000..c9d9ab2b --- /dev/null +++ b/tools/tests/template @@ -0,0 +1,25 @@ +trap cleanup EXIT + +create_files +submit_jobs +wait_for_jobs + +# Ignore slurmstepd errors +if [ -f $output ]; then + sed '/rmdir(\/sys\/fs\/cgroup/d' -i $output + sed '/Cannot open \/proc\/fs\/lustre/d' -i $output +fi + +# Any output means error +if [ -s $output ]; then + cat $output + exit 1 +fi + +if [ ! -f $scratch/$outfile ]; then + echo Stage out file does not exist + exit 1 +elif [ -s $scratch/$outfile ]; then + echo Stage out file is empty + exit 1 +fi