From ca9b1d4f28a16f663d7f486fa114a38a7d570361 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 12 Dec 2024 11:15:03 +0100 Subject: [PATCH 1/4] fix handling of quotes in rpath wrapper In some software the compiler must be invoked with an argument containing a literal single quote which requires escaping in the call. E.g.: `gcc -DFOO=\'value\'` However the current rpath wrapper would remove those single quotes causing errors during compilation or erronous behavior of the compiled binary. Fix by replacing the `eval` of the `rpath_args.py` output by `readarray -t`. An array asignment could be used (`CMD_ARGS=( $(rpath_args.py ...) )`) but that would break up arguments containing spaces. Quotes from the output are kept literally, so quoting to avoid this is not possible. `readarray -t` works and is widely available. Also some minor fixes related to quoting in the bash script template. --- easybuild/scripts/rpath_args.py | 17 +++-- .../scripts/rpath_wrapper_template.sh.in | 23 +++--- test/framework/toolchain.py | 70 ++++++++++++------- 3 files changed, 63 insertions(+), 47 deletions(-) diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py index b477aa63d8..ddf8234a49 100755 --- a/easybuild/scripts/rpath_args.py +++ b/easybuild/scripts/rpath_args.py @@ -25,11 +25,10 @@ ## """ Utility script used by RPATH wrapper script; -output is statements that define the following environment variables -* $CMD_ARGS: new list of command line arguments to pass -* $RPATH_ARGS: command line option to specify list of paths to RPATH +output is a list of arguments to be passed to the wrapped command, one per line author: Kenneth Hoste (HPC-UGent) +author: Alexander Grund (TU Dresden) """ import os import re @@ -161,8 +160,12 @@ def is_new_existing_path(new_path, paths): # add -rpath flags in front cmd_args = cmd_args_rpath + cmd_args -# wrap all arguments into single quotes to avoid further bash expansion -cmd_args = ["'%s'" % a.replace("'", "''") for a in cmd_args] -# output: statement to define $CMD_ARGS and $RPATH_ARGS -print("CMD_ARGS=(%s)" % ' '.join(cmd_args)) +def quote(arg): + """Quote the argument for use in a shell to avoid further expansion""" + # Wrap in single quotes and replace every single quote by '"'"' + # This terminates the single-quoted string, inserts a literal single quote and starts a new single-quotest-string + return "'" + arg.replace("'", "'\"'\"'") + "'" + + +print('\n'.join(cmd_args)) diff --git a/easybuild/scripts/rpath_wrapper_template.sh.in b/easybuild/scripts/rpath_wrapper_template.sh.in index 44a5aac43a..6ac6155d60 100644 --- a/easybuild/scripts/rpath_wrapper_template.sh.in +++ b/easybuild/scripts/rpath_wrapper_template.sh.in @@ -30,6 +30,7 @@ # before actually calling the original compiler/linker command. # # author: Kenneth Hoste (HPC-UGent) +# author: Alexander Grund (TU Dresden) set -e @@ -37,13 +38,13 @@ set -e function log { # escape percent signs, since this is a template script # that will templated using Python string templating - echo "($$) [$(date "+%%Y-%%m-%%d %%H:%%M:%%S")] $1" >> %(rpath_wrapper_log)s + echo "($$) [$(date "+%%Y-%%m-%%d %%H:%%M:%%S")] $1" >> '%(rpath_wrapper_log)s' } # command name -CMD=`basename $0` +CMD=$(basename $0) -log "found CMD: $CMD | original command: %(orig_cmd)s | orig args: '$(echo \"$@\")'" +log "found CMD: $CMD | original command: %(orig_cmd)s | orig args: $*)" # rpath_args.py script spits out statement that defines $CMD_ARGS # options for 'python' command (see https://docs.python.org/3/using/cmdline.html#miscellaneous-options) @@ -52,18 +53,12 @@ log "found CMD: $CMD | original command: %(orig_cmd)s | orig args: '$(echo \"$@\ # * -s: don’t add the user site-packages directory to sys.path; # * -S: disable the import of the module site and the site-dependent manipulations of sys.path that it entails; # (once we only support Python 3, we can (also) use -I (isolated mode) -log "%(python)s -E -O -s -S %(rpath_args_py)s $CMD '%(rpath_filter)s' '%(rpath_include)s' $(echo \"$@\")'" -rpath_args_out=$(%(python)s -E -O -s -S %(rpath_args_py)s $CMD '%(rpath_filter)s' '%(rpath_include)s' "$@") - -log "rpath_args_out: -$rpath_args_out" - -# define $CMD_ARGS by evaluating output of rpath_args.py script -eval $rpath_args_out +log "%(python)s -E -O -s -S %(rpath_args_py)s $CMD '%(rpath_filter)s' '%(rpath_include)s' $*" +readarray -t CMD_ARGS < <( %(python)s -E -O -s -S %(rpath_args_py)s $CMD '%(rpath_filter)s' '%(rpath_include)s' "$@" ) # exclude location of this wrapper from $PATH to avoid other potential wrappers calling this wrapper -export PATH=$(echo $PATH | tr ':' '\n' | grep -v "^%(wrapper_dir)s$" | tr '\n' ':') +export PATH=$(echo "$PATH" | tr ':' '\n' | grep -v "^%(wrapper_dir)s$" | tr '\n' ':') # call original command with modified list of command line arguments -log "running '%(orig_cmd)s $(echo ${CMD_ARGS[@]})'" -%(orig_cmd)s "${CMD_ARGS[@]}" +log "running: %(orig_cmd)s $(echo "${CMD_ARGS[@]}")" +'%(orig_cmd)s' "${CMD_ARGS[@]}" diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index beeb2e3512..6711cacbd0 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -2744,9 +2744,12 @@ def test_rpath_args_script(self): def test_toolchain_prepare_rpath(self): """Test toolchain.prepare under --rpath""" + # Code for a bash script that prints each passed argument on a new line + BASH_SCRIPT_PRINT_ARGS = '#!/bin/bash\nfor arg in "$@"; do echo "$arg"; done' + # put fake 'g++' command in place that just echos its arguments fake_gxx = os.path.join(self.test_prefix, 'fake', 'g++') - write_file(fake_gxx, '#!/bin/bash\necho "$@"') + write_file(fake_gxx, BASH_SCRIPT_PRINT_ARGS) adjust_permissions(fake_gxx, stat.S_IXUSR) os.environ['PATH'] = '%s:%s' % (os.path.join(self.test_prefix, 'fake'), os.getenv('PATH', '')) @@ -2789,7 +2792,7 @@ def test_toolchain_prepare_rpath(self): # Check that we can create a wrapper for a toolchain for which self.compilers() returns 'None' for the Fortran # compilers (i.e. Clang) fake_clang = os.path.join(self.test_prefix, 'fake', 'clang') - write_file(fake_clang, '#!/bin/bash\necho "$@"') + write_file(fake_clang, BASH_SCRIPT_PRINT_ARGS) adjust_permissions(fake_clang, stat.S_IXUSR) tc_clang = Clang(name='Clang', version='1') tc_clang.prepare_rpath_wrappers() @@ -2852,18 +2855,25 @@ def test_toolchain_prepare_rpath(self): '-L%s/foo' % self.test_prefix, '-L/bar', "'$FOO'", - '-DX="\\"\\""', + # C/C++ preprocessor value including a quotes + r'-DX1="\"\""', + r'-DX2="\"I\""', + r"-DY1=\'\'", + r"-DY2=\'J\'", ]) out, ec = run_cmd(cmd) self.assertEqual(ec, 0) - expected = ' '.join([ + expected = '\n'.join([ '-Wl,--disable-new-dtags', '-Wl,-rpath=%s/foo' % self.test_prefix, '%(user)s.c', '-L%s/foo' % self.test_prefix, '-L/bar', '$FOO', - '-DX=""', + '-DX1=""', + '-DX2="I"', + "-DY1=''", + "-DY2='J'", ]) self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')}) @@ -2888,35 +2898,43 @@ def test_toolchain_prepare_rpath(self): for path in paths: mkdir(path, parents=True) args = ['-L%s' % x for x in paths] + path_with_spaces = os.path.join(self.test_prefix, 'prefix/with spaces') + mkdir(path_with_spaces) + args.append('-L"%s"' % path_with_spaces) cmd = "g++ ${USER}.c %s" % ' '.join(args) out, ec = run_cmd(cmd, simple=False) self.assertEqual(ec, 0) - expected = ' '.join([ + expected = '\n'.join([ '-Wl,--disable-new-dtags', - '-Wl,-rpath=%s/tmp/foo/' % self.test_prefix, - '-Wl,-rpath=%s/prefix/software/stubs/1.2.3/lib' % self.test_prefix, - '-Wl,-rpath=%s/prefix/software/foobar/4.5/notreallystubs' % self.test_prefix, - '-Wl,-rpath=%s/prefix/software/zlib/1.2.11/lib' % self.test_prefix, - '-Wl,-rpath=%s/prefix/software/foobar/4.5/stubsbutnotreally' % self.test_prefix, + '-Wl,-rpath=%(libdir)s/tmp/foo/', + '-Wl,-rpath=%(libdir)s/prefix/software/stubs/1.2.3/lib', + '-Wl,-rpath=%(libdir)s/prefix/software/foobar/4.5/notreallystubs', + '-Wl,-rpath=%(libdir)s/prefix/software/zlib/1.2.11/lib', + '-Wl,-rpath=%(libdir)s/prefix/software/foobar/4.5/stubsbutnotreally', + '-Wl,-rpath=%(path_with_spaces)s', '%(user)s.c', - '-L%s/prefix/software/CUDA/1.2.3/lib/stubs/' % self.test_prefix, - '-L%s/prefix/software/CUDA/1.2.3/stubs/lib/' % self.test_prefix, - '-L%s/tmp/foo/' % self.test_prefix, - '-L%s/prefix/software/stubs/1.2.3/lib' % self.test_prefix, - '-L%s/prefix/software/CUDA/1.2.3/lib/stubs' % self.test_prefix, - '-L%s/prefix/software/CUDA/1.2.3/stubs/lib' % self.test_prefix, - '-L%s/prefix/software/CUDA/1.2.3/lib64/stubs/' % self.test_prefix, - '-L%s/prefix/software/CUDA/1.2.3/stubs/lib64/' % self.test_prefix, - '-L%s/prefix/software/foobar/4.5/notreallystubs' % self.test_prefix, - '-L%s/prefix/software/CUDA/1.2.3/lib64/stubs' % self.test_prefix, - '-L%s/prefix/software/CUDA/1.2.3/stubs/lib64' % self.test_prefix, - '-L%s/prefix/software/zlib/1.2.11/lib' % self.test_prefix, - '-L%s/prefix/software/bleh/0/lib/stubs' % self.test_prefix, - '-L%s/prefix/software/foobar/4.5/stubsbutnotreally' % self.test_prefix, + '-L%(libdir)s/prefix/software/CUDA/1.2.3/lib/stubs/', + '-L%(libdir)s/prefix/software/CUDA/1.2.3/stubs/lib/', + '-L%(libdir)s/tmp/foo/', + '-L%(libdir)s/prefix/software/stubs/1.2.3/lib', + '-L%(libdir)s/prefix/software/CUDA/1.2.3/lib/stubs', + '-L%(libdir)s/prefix/software/CUDA/1.2.3/stubs/lib', + '-L%(libdir)s/prefix/software/CUDA/1.2.3/lib64/stubs/', + '-L%(libdir)s/prefix/software/CUDA/1.2.3/stubs/lib64/', + '-L%(libdir)s/prefix/software/foobar/4.5/notreallystubs', + '-L%(libdir)s/prefix/software/CUDA/1.2.3/lib64/stubs', + '-L%(libdir)s/prefix/software/CUDA/1.2.3/stubs/lib64', + '-L%(libdir)s/prefix/software/zlib/1.2.11/lib', + '-L%(libdir)s/prefix/software/bleh/0/lib/stubs', + '-L%(libdir)s/prefix/software/foobar/4.5/stubsbutnotreally', + '-L%(path_with_spaces)s', ]) - self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')}) + self.assertEqual(out.strip(), expected % {'libdir': self.test_prefix, + 'user': os.getenv('USER'), + 'path_with_spaces': path_with_spaces, + }) # calling prepare() again should *not* result in wrapping the existing RPATH wrappers # this can happen when building extensions From 7981f47e8b9de5d502073076ea08704c7d3fd41e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 12 Dec 2024 13:30:37 +0100 Subject: [PATCH 2/4] Fix tests Tests expected output of the rpath Python script to be space separated but now it is line separated. Also the output does not include the name of the array anymore. To aid with debugging a helper method is introduced and the output is compared as a list with one line each. --- test/framework/toolchain.py | 481 ++++++++++++++++++------------------ test/framework/toy_build.py | 17 +- 2 files changed, 240 insertions(+), 258 deletions(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 6711cacbd0..b1e568f51a 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -82,6 +82,15 @@ def tearDown(self): st.get_cpu_vendor = self.orig_get_cpu_vendor super(ToolchainTest, self).tearDown() + def run_cmd_and_split(self, cmd): + """Run the command, check return code and return output as list of lines""" + out, ec = run_cmd(cmd, simple=False) + self.assertEqual(ec, 0) + # Remove 1 trailing newline + if out: + self.assertEqual(out[-1], '\n') + return out[:-1].split('\n') + def get_toolchain(self, name, version=None): """Get a toolchain object instance to test with.""" tc_class, _ = search_toolchain(name) @@ -2315,145 +2324,136 @@ def test_rpath_args_script(self): ]) # simplest possible compiler command - out, ec = run_cmd("%s gcc '' '%s' -c foo.c" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + out = self.run_cmd_and_split("%s gcc '' '%s' -c foo.c" % (script, rpath_inc)) cmd_args = [ - "'-Wl,-rpath=%s/lib'" % self.test_prefix, - "'-Wl,-rpath=%s/lib64'" % self.test_prefix, - "'-Wl,-rpath=$ORIGIN'", - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", - "'-Wl,--disable-new-dtags'", - "'-c'", - "'foo.c'", + "-Wl,-rpath=%s/lib" % self.test_prefix, + "-Wl,-rpath=%s/lib64" % self.test_prefix, + "-Wl,-rpath=$ORIGIN", + "-Wl,-rpath=$ORIGIN/../lib", + "-Wl,-rpath=$ORIGIN/../lib64", + "-Wl,--disable-new-dtags", + "-c", + "foo.c", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(out, cmd_args) # linker command, --enable-new-dtags should be replaced with --disable-new-dtags - out, ec = run_cmd("%s ld '' '%s' --enable-new-dtags foo.o" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + out = self.run_cmd_and_split("%s ld '' '%s' --enable-new-dtags foo.o" % (script, rpath_inc)) cmd_args = [ - "'-rpath=%s/lib'" % self.test_prefix, - "'-rpath=%s/lib64'" % self.test_prefix, - "'-rpath=$ORIGIN'", - "'-rpath=$ORIGIN/../lib'", - "'-rpath=$ORIGIN/../lib64'", - "'--disable-new-dtags'", - "'--disable-new-dtags'", - "'foo.o'", + "-rpath=%s/lib" % self.test_prefix, + "-rpath=%s/lib64" % self.test_prefix, + "-rpath=$ORIGIN", + "-rpath=$ORIGIN/../lib", + "-rpath=$ORIGIN/../lib64", + "--disable-new-dtags", + "--disable-new-dtags", + "foo.o", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(out, cmd_args) # compiler command, -Wl,--enable-new-dtags should be replaced with -Wl,--disable-new-dtags - out, ec = run_cmd("%s gcc '' '%s' -Wl,--enable-new-dtags foo.c" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + out = self.run_cmd_and_split("%s gcc '' '%s' -Wl,--enable-new-dtags foo.c" % (script, rpath_inc)) cmd_args = [ - "'-Wl,-rpath=%s/lib'" % self.test_prefix, - "'-Wl,-rpath=%s/lib64'" % self.test_prefix, - "'-Wl,-rpath=$ORIGIN'", - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", - "'-Wl,--disable-new-dtags'", - "'-Wl,--disable-new-dtags'", - "'foo.c'", + "-Wl,-rpath=%s/lib" % self.test_prefix, + "-Wl,-rpath=%s/lib64" % self.test_prefix, + "-Wl,-rpath=$ORIGIN", + "-Wl,-rpath=$ORIGIN/../lib", + "-Wl,-rpath=$ORIGIN/../lib64", + "-Wl,--disable-new-dtags", + "-Wl,--disable-new-dtags", + "foo.c", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(out, cmd_args) # test passing no arguments - out, ec = run_cmd("%s gcc '' '%s'" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + out = self.run_cmd_and_split("%s gcc '' '%s'" % (script, rpath_inc)) cmd_args = [ - "'-Wl,-rpath=%s/lib'" % self.test_prefix, - "'-Wl,-rpath=%s/lib64'" % self.test_prefix, - "'-Wl,-rpath=$ORIGIN'", - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", - "'-Wl,--disable-new-dtags'", + "-Wl,-rpath=%s/lib" % self.test_prefix, + "-Wl,-rpath=%s/lib64" % self.test_prefix, + "-Wl,-rpath=$ORIGIN", + "-Wl,-rpath=$ORIGIN/../lib", + "-Wl,-rpath=$ORIGIN/../lib64", + "-Wl,--disable-new-dtags", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(out, cmd_args) # test passing a single empty argument - out, ec = run_cmd("%s ld.gold '' '%s' ''" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + out = self.run_cmd_and_split("%s ld.gold '' '%s' ''" % (script, rpath_inc)) cmd_args = [ - "'-rpath=%s/lib'" % self.test_prefix, - "'-rpath=%s/lib64'" % self.test_prefix, - "'-rpath=$ORIGIN'", - "'-rpath=$ORIGIN/../lib'", - "'-rpath=$ORIGIN/../lib64'", - "'--disable-new-dtags'", - "''", + "-rpath=%s/lib" % self.test_prefix, + "-rpath=%s/lib64" % self.test_prefix, + "-rpath=$ORIGIN", + "-rpath=$ORIGIN/../lib", + "-rpath=$ORIGIN/../lib64", + "--disable-new-dtags", + "", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(out, cmd_args) # single -L argument, but non-existing path => not used in RPATH, but -L option is retained cmd = "%s gcc '' '%s' foo.c -L%s/foo -lfoo" % (script, rpath_inc, self.test_prefix) - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + out = self.run_cmd_and_split(cmd) cmd_args = [ - "'-Wl,-rpath=%s/lib'" % self.test_prefix, - "'-Wl,-rpath=%s/lib64'" % self.test_prefix, - "'-Wl,-rpath=$ORIGIN'", - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", - "'-Wl,--disable-new-dtags'", - "'foo.c'", - "'-L%s/foo'" % self.test_prefix, - "'-lfoo'", + "-Wl,-rpath=%s/lib" % self.test_prefix, + "-Wl,-rpath=%s/lib64" % self.test_prefix, + "-Wl,-rpath=$ORIGIN", + "-Wl,-rpath=$ORIGIN/../lib", + "-Wl,-rpath=$ORIGIN/../lib64", + "-Wl,--disable-new-dtags", + "foo.c", + "-L%s/foo" % self.test_prefix, + "-lfoo", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(out, cmd_args) # single -L argument again, with existing path mkdir(os.path.join(self.test_prefix, 'foo')) - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + out = self.run_cmd_and_split(cmd) cmd_args = [ - "'-Wl,-rpath=%s/lib'" % self.test_prefix, - "'-Wl,-rpath=%s/lib64'" % self.test_prefix, - "'-Wl,-rpath=$ORIGIN'", - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", - "'-Wl,--disable-new-dtags'", - "'-Wl,-rpath=%s/foo'" % self.test_prefix, - "'foo.c'", - "'-L%s/foo'" % self.test_prefix, - "'-lfoo'", + "-Wl,-rpath=%s/lib" % self.test_prefix, + "-Wl,-rpath=%s/lib64" % self.test_prefix, + "-Wl,-rpath=$ORIGIN", + "-Wl,-rpath=$ORIGIN/../lib", + "-Wl,-rpath=$ORIGIN/../lib64", + "-Wl,--disable-new-dtags", + "-Wl,-rpath=%s/foo" % self.test_prefix, + "foo.c", + "-L%s/foo" % self.test_prefix, + "-lfoo", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(out, cmd_args) # relative paths passed to -L are *not* RPATH'ed in - out, ec = run_cmd("%s gcc '' '%s' foo.c -L../lib -lfoo" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + out = self.run_cmd_and_split("%s gcc '' '%s' foo.c -L../lib -lfoo" % (script, rpath_inc)) cmd_args = [ - "'-Wl,-rpath=%s/lib'" % self.test_prefix, - "'-Wl,-rpath=%s/lib64'" % self.test_prefix, - "'-Wl,-rpath=$ORIGIN'", - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", - "'-Wl,--disable-new-dtags'", - "'foo.c'", - "'-L../lib'", - "'-lfoo'", + "-Wl,-rpath=%s/lib" % self.test_prefix, + "-Wl,-rpath=%s/lib64" % self.test_prefix, + "-Wl,-rpath=$ORIGIN", + "-Wl,-rpath=$ORIGIN/../lib", + "-Wl,-rpath=$ORIGIN/../lib64", + "-Wl,--disable-new-dtags", + "foo.c", + "-L../lib", + "-lfoo", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(out, cmd_args) # single -L argument, with value separated by a space cmd = "%s gcc '' '%s' foo.c -L %s/foo -lfoo" % (script, rpath_inc, self.test_prefix) - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + out = self.run_cmd_and_split(cmd) cmd_args = [ - "'-Wl,-rpath=%s/lib'" % self.test_prefix, - "'-Wl,-rpath=%s/lib64'" % self.test_prefix, - "'-Wl,-rpath=$ORIGIN'", - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", - "'-Wl,--disable-new-dtags'", - "'-Wl,-rpath=%s/foo'" % self.test_prefix, - "'foo.c'", - "'-L%s/foo'" % self.test_prefix, - "'-lfoo'", + "-Wl,-rpath=%s/lib" % self.test_prefix, + "-Wl,-rpath=%s/lib64" % self.test_prefix, + "-Wl,-rpath=$ORIGIN", + "-Wl,-rpath=$ORIGIN/../lib", + "-Wl,-rpath=$ORIGIN/../lib64", + "-Wl,--disable-new-dtags", + "-Wl,-rpath=%s/foo" % self.test_prefix, + "foo.c", + "-L%s/foo" % self.test_prefix, + "-lfoo", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(out, cmd_args) mkdir(os.path.join(self.test_prefix, 'bar')) mkdir(os.path.join(self.test_prefix, 'lib64')) @@ -2474,29 +2474,28 @@ def test_rpath_args_script(self): '-L/usr/lib', '-L%s/bar' % self.test_prefix, ]) - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + out = self.run_cmd_and_split(cmd) cmd_args = [ - "'-rpath=%s/lib'" % self.test_prefix, - "'-rpath=%s/lib64'" % self.test_prefix, - "'-rpath=$ORIGIN'", - "'-rpath=$ORIGIN/../lib'", - "'-rpath=$ORIGIN/../lib64'", - "'--disable-new-dtags'", - "'-rpath=%s/foo'" % self.test_prefix, - "'-rpath=%s/lib64'" % self.test_prefix, - "'-rpath=/usr/lib'", - "'-rpath=%s/bar'" % self.test_prefix, - "'-L%s/foo'" % self.test_prefix, - "'foo.o'", - "'-L%s/lib64'" % self.test_prefix, - "'-L%s/foo'" % self.test_prefix, - "'-lfoo'", - "'-lbar'", - "'-L/usr/lib'", - "'-L%s/bar'" % self.test_prefix, + "-rpath=%s/lib" % self.test_prefix, + "-rpath=%s/lib64" % self.test_prefix, + "-rpath=$ORIGIN", + "-rpath=$ORIGIN/../lib", + "-rpath=$ORIGIN/../lib64", + "--disable-new-dtags", + "-rpath=%s/foo" % self.test_prefix, + "-rpath=%s/lib64" % self.test_prefix, + "-rpath=/usr/lib", + "-rpath=%s/bar" % self.test_prefix, + "-L%s/foo" % self.test_prefix, + "foo.o", + "-L%s/lib64" % self.test_prefix, + "-L%s/foo" % self.test_prefix, + "-lfoo", + "-lbar", + "-L/usr/lib", + "-L%s/bar" % self.test_prefix, ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(out, cmd_args) # test specifying of custom rpath filter cmd = ' '.join([ @@ -2511,24 +2510,23 @@ def test_rpath_args_script(self): '-L/bar', '-lbar', ]) - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + out = self.run_cmd_and_split(cmd) cmd_args = [ - "'-rpath=%s/lib'" % self.test_prefix, - "'-rpath=%s/lib64'" % self.test_prefix, - "'-rpath=$ORIGIN'", - "'-rpath=$ORIGIN/../lib'", - "'-rpath=$ORIGIN/../lib64'", - "'--disable-new-dtags'", - "'-rpath=%s/lib64'" % self.test_prefix, - "'-L/foo'", - "'foo.o'", - "'-L%s/lib64'" % self.test_prefix, - "'-lfoo'", - "'-L/bar'", - "'-lbar'", + "-rpath=%s/lib" % self.test_prefix, + "-rpath=%s/lib64" % self.test_prefix, + "-rpath=$ORIGIN", + "-rpath=$ORIGIN/../lib", + "-rpath=$ORIGIN/../lib64", + "--disable-new-dtags", + "-rpath=%s/lib64" % self.test_prefix, + "-L/foo", + "foo.o", + "-L%s/lib64" % self.test_prefix, + "-lfoo", + "-L/bar", + "-lbar", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(out, cmd_args) # slightly trimmed down real-life example (compilation of XZ) for subdir in ['icc/lib/intel64', 'imkl/lib', 'imkl/mkl/lib/intel64', 'gettext/lib']: @@ -2550,35 +2548,34 @@ def test_rpath_args_script(self): '-Wl,-rpath', '-Wl,/example/software/XZ/5.2.2-intel-2016b/lib', ]) - out, ec = run_cmd("%s icc '' '%s' %s" % (script, rpath_inc, args), simple=False) - self.assertEqual(ec, 0) + out = self.run_cmd_and_split("%s icc '' '%s' %s" % (script, rpath_inc, args)) cmd_args = [ - "'-Wl,-rpath=%s/lib'" % self.test_prefix, - "'-Wl,-rpath=%s/lib64'" % self.test_prefix, - "'-Wl,-rpath=$ORIGIN'", - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", - "'-Wl,--disable-new-dtags'", - "'-Wl,-rpath=%s/icc/lib/intel64'" % self.test_prefix, - "'-Wl,-rpath=%s/imkl/lib'" % self.test_prefix, - "'-Wl,-rpath=%s/imkl/mkl/lib/intel64'" % self.test_prefix, - "'-Wl,-rpath=%s/gettext/lib'" % self.test_prefix, - "'-fvisibility=hidden'", - "'-Wall'", - "'-O2'", - "'-xHost'", - "'-o' '.libs/lzmainfo'", - "'lzmainfo-lzmainfo.o' 'lzmainfo-tuklib_progname.o' 'lzmainfo-tuklib_exit.o'", - "'-L%s/icc/lib/intel64'" % self.test_prefix, - "'-L%s/imkl/lib'" % self.test_prefix, - "'-L%s/imkl/mkl/lib/intel64'" % self.test_prefix, - "'-L%s/gettext/lib'" % self.test_prefix, - "'../../src/liblzma/.libs/liblzma.so'", - "'-lrt' '-liomp5' '-lpthread'", - "'-Wl,-rpath'", - "'-Wl,/example/software/XZ/5.2.2-intel-2016b/lib'", + "-Wl,-rpath=%s/lib" % self.test_prefix, + "-Wl,-rpath=%s/lib64" % self.test_prefix, + "-Wl,-rpath=$ORIGIN", + "-Wl,-rpath=$ORIGIN/../lib", + "-Wl,-rpath=$ORIGIN/../lib64", + "-Wl,--disable-new-dtags", + "-Wl,-rpath=%s/icc/lib/intel64" % self.test_prefix, + "-Wl,-rpath=%s/imkl/lib" % self.test_prefix, + "-Wl,-rpath=%s/imkl/mkl/lib/intel64" % self.test_prefix, + "-Wl,-rpath=%s/gettext/lib" % self.test_prefix, + "-fvisibility=hidden", + "-Wall", + "-O2", + "-xHost", + "-o", ".libs/lzmainfo", + "lzmainfo-lzmainfo.o", "lzmainfo-tuklib_progname.o", "lzmainfo-tuklib_exit.o", + "-L%s/icc/lib/intel64" % self.test_prefix, + "-L%s/imkl/lib" % self.test_prefix, + "-L%s/imkl/mkl/lib/intel64" % self.test_prefix, + "-L%s/gettext/lib" % self.test_prefix, + "../../src/liblzma/.libs/liblzma.so", + "-lrt", "-liomp5", "-lpthread", + "-Wl,-rpath", + "-Wl,/example/software/XZ/5.2.2-intel-2016b/lib", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(out, cmd_args) # trimmed down real-life example involving quotes and escaped quotes (compilation of GCC) args = [ @@ -2594,35 +2591,33 @@ def test_rpath_args_script(self): '../../gcc/version.c', ] cmd = "%s g++ '' '%s' %s" % (script, rpath_inc, ' '.join(args)) - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + out = self.run_cmd_and_split(cmd) cmd_args = [ - "'-Wl,-rpath=%s/lib'" % self.test_prefix, - "'-Wl,-rpath=%s/lib64'" % self.test_prefix, - "'-Wl,-rpath=$ORIGIN'", - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", - "'-Wl,--disable-new-dtags'", - "'-DHAVE_CONFIG_H'", - "'-I.'", - "'-Ibuild'", - "'-I../../gcc'", - "'-DBASEVER=\"5.4.0\"'", - "'-DDATESTAMP=\"\"'", - "'-DPKGVERSION=\"(GCC) \"'", - "'-DBUGURL=\"\"'", - "'-o' 'build/version.o'", - "'../../gcc/version.c'", + "-Wl,-rpath=%s/lib" % self.test_prefix, + "-Wl,-rpath=%s/lib64" % self.test_prefix, + "-Wl,-rpath=$ORIGIN", + "-Wl,-rpath=$ORIGIN/../lib", + "-Wl,-rpath=$ORIGIN/../lib64", + "-Wl,--disable-new-dtags", + "-DHAVE_CONFIG_H", + "-I.", + "-Ibuild", + "-I../../gcc", + "-DBASEVER=\"5.4.0\"", + "-DDATESTAMP=\"\"", + "-DPKGVERSION=\"(GCC) \"", + "-DBUGURL=\"\"", + "-o", "build/version.o", + "../../gcc/version.c", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(out, cmd_args) # verify that no -rpath arguments are injected when command is run in 'version check' mode for extra_args in ["-v", "-V", "--version", "-dumpversion", "-v -L/test/lib"]: cmd = "%s g++ '' '%s' %s" % (script, rpath_inc, extra_args) - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(["'%s'" % x for x in extra_args.split(' ')])) + out = self.run_cmd_and_split(cmd) + self.assertEqual(out, extra_args.split(' ')) # if a compiler command includes "-x c++-header" or "-x c-header" (which imply no linking is done), # we should *not* inject -Wl,-rpath options, since those enable linking as a side-effect; @@ -2634,24 +2629,23 @@ def test_rpath_args_script(self): ] for extra_args in test_cases: cmd = "%s g++ '' '%s' foo.c -O2 %s" % (script, rpath_inc, extra_args) - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) - cmd_args = ["'foo.c'", "'-O2'"] + ["'%s'" % x for x in extra_args.split(' ')] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + out = self.run_cmd_and_split(cmd) + cmd_args = ["foo.c", "-O2"] + extra_args.split(' ') + self.assertEqual(out, cmd_args) # check whether $LIBRARY_PATH is taken into account test_cmd_gcc = "%s gcc '' '%s' -c foo.c" % (script, rpath_inc) pre_cmd_args_gcc = [ - "'-Wl,-rpath=%s/lib'" % self.test_prefix, - "'-Wl,-rpath=%s/lib64'" % self.test_prefix, - "'-Wl,-rpath=$ORIGIN'", - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", - "'-Wl,--disable-new-dtags'", + "-Wl,-rpath=%s/lib" % self.test_prefix, + "-Wl,-rpath=%s/lib64" % self.test_prefix, + "-Wl,-rpath=$ORIGIN", + "-Wl,-rpath=$ORIGIN/../lib", + "-Wl,-rpath=$ORIGIN/../lib64", + "-Wl,--disable-new-dtags", ] post_cmd_args_gcc = [ - "'-c'", - "'foo.c'", + "-c", + "foo.c", ] test_cmd_ld = ' '.join([ @@ -2668,25 +2662,25 @@ def test_rpath_args_script(self): '-L%s/bar' % self.test_prefix, ]) pre_cmd_args_ld = [ - "'-rpath=%s/lib'" % self.test_prefix, - "'-rpath=%s/lib64'" % self.test_prefix, - "'-rpath=$ORIGIN'", - "'-rpath=$ORIGIN/../lib'", - "'-rpath=$ORIGIN/../lib64'", - "'--disable-new-dtags'", - "'-rpath=%s/foo'" % self.test_prefix, - "'-rpath=%s/lib64'" % self.test_prefix, - "'-rpath=/usr/lib'", - "'-rpath=%s/bar'" % self.test_prefix, + "-rpath=%s/lib" % self.test_prefix, + "-rpath=%s/lib64" % self.test_prefix, + "-rpath=$ORIGIN", + "-rpath=$ORIGIN/../lib", + "-rpath=$ORIGIN/../lib64", + "--disable-new-dtags", + "-rpath=%s/foo" % self.test_prefix, + "-rpath=%s/lib64" % self.test_prefix, + "-rpath=/usr/lib", + "-rpath=%s/bar" % self.test_prefix, ] post_cmd_args_ld = [ - "'-L%s/foo'" % self.test_prefix, - "'foo.o'", - "'-L%s/lib64'" % self.test_prefix, - "'-lfoo'", - "'-lbar'", - "'-L/usr/lib'", - "'-L%s/bar'" % self.test_prefix, + "-L%s/foo" % self.test_prefix, + "foo.o", + "-L%s/lib64" % self.test_prefix, + "-lfoo", + "-lbar", + "-L/usr/lib", + "-L%s/bar" % self.test_prefix, ] library_paths = [ @@ -2702,15 +2696,13 @@ def test_rpath_args_script(self): os.environ['LIBRARY_PATH'] = ':'.join(library_path) - out, ec = run_cmd(test_cmd_gcc, simple=False) - self.assertEqual(ec, 0) - cmd_args = pre_cmd_args_gcc + ["'-Wl,-rpath=%s'" % x for x in library_path if x] + post_cmd_args_gcc - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + out = self.run_cmd_and_split(test_cmd_gcc) + cmd_args = pre_cmd_args_gcc + ["-Wl,-rpath=%s" % x for x in library_path if x] + post_cmd_args_gcc + self.assertEqual(out, cmd_args) - out, ec = run_cmd(test_cmd_ld, simple=False) - self.assertEqual(ec, 0) - cmd_args = pre_cmd_args_ld + ["'-rpath=%s'" % x for x in library_path if x] + post_cmd_args_ld - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + out = self.run_cmd_and_split(test_cmd_ld) + cmd_args = pre_cmd_args_ld + ["-rpath=%s" % x for x in library_path if x] + post_cmd_args_ld + self.assertEqual(out, cmd_args) # paths already listed via -L don't get included again as RPATH option new_lib64 = os.path.join(self.test_prefix, 'new', 'lib64') @@ -2728,18 +2720,16 @@ def test_rpath_args_script(self): ] os.environ['LIBRARY_PATH'] = ':'.join(library_path) - out, ec = run_cmd(test_cmd_gcc, simple=False) - self.assertEqual(ec, 0) + out = self.run_cmd_and_split(test_cmd_gcc) # no -L options in GCC command, so all $LIBRARY_PATH entries are retained except for last one (lib symlink) - cmd_args = pre_cmd_args_gcc + ["'-Wl,-rpath=%s'" % x for x in library_path[:-1] if x] + post_cmd_args_gcc - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + cmd_args = pre_cmd_args_gcc + ["-Wl,-rpath=%s" % x for x in library_path[:-1] if x] + post_cmd_args_gcc + self.assertEqual(out, cmd_args) - out, ec = run_cmd(test_cmd_ld, simple=False) - self.assertEqual(ec, 0) + out = self.run_cmd_and_split(test_cmd_ld) # only new path from $LIBRARY_PATH is included as -rpath option, # since others are already included via corresponding -L flag - cmd_args = pre_cmd_args_ld + ["'-rpath=%s'" % new_lib64] + post_cmd_args_ld - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + cmd_args = pre_cmd_args_ld + ["-rpath=%s" % new_lib64] + post_cmd_args_ld + self.assertEqual(out, cmd_args) def test_toolchain_prepare_rpath(self): """Test toolchain.prepare under --rpath""" @@ -2855,27 +2845,28 @@ def test_toolchain_prepare_rpath(self): '-L%s/foo' % self.test_prefix, '-L/bar', "'$FOO'", + "''", # Empty argument # C/C++ preprocessor value including a quotes r'-DX1="\"\""', r'-DX2="\"I\""', r"-DY1=\'\'", r"-DY2=\'J\'", ]) - out, ec = run_cmd(cmd) - self.assertEqual(ec, 0) - expected = '\n'.join([ + out = self.run_cmd_and_split(cmd) + expected = [ '-Wl,--disable-new-dtags', '-Wl,-rpath=%s/foo' % self.test_prefix, - '%(user)s.c', + '%s.c' % os.getenv('USER'), '-L%s/foo' % self.test_prefix, '-L/bar', '$FOO', + '', '-DX1=""', '-DX2="I"', "-DY1=''", "-DY2='J'", - ]) - self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')}) + ] + self.assertEqual(out, expected) # check whether 'stubs' library directory are correctly filtered out paths = [ @@ -2903,10 +2894,9 @@ def test_toolchain_prepare_rpath(self): args.append('-L"%s"' % path_with_spaces) cmd = "g++ ${USER}.c %s" % ' '.join(args) - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + out = self.run_cmd_and_split(cmd) - expected = '\n'.join([ + expected = ('\n'.join([ '-Wl,--disable-new-dtags', '-Wl,-rpath=%(libdir)s/tmp/foo/', '-Wl,-rpath=%(libdir)s/prefix/software/stubs/1.2.3/lib', @@ -2930,11 +2920,8 @@ def test_toolchain_prepare_rpath(self): '-L%(libdir)s/prefix/software/bleh/0/lib/stubs', '-L%(libdir)s/prefix/software/foobar/4.5/stubsbutnotreally', '-L%(path_with_spaces)s', - ]) - self.assertEqual(out.strip(), expected % {'libdir': self.test_prefix, - 'user': os.getenv('USER'), - 'path_with_spaces': path_with_spaces, - }) + ]) % {'libdir': self.test_prefix, 'user': os.getenv('USER'), 'path_with_spaces': path_with_spaces}).split('\n') + self.assertEqual(out, expected) # calling prepare() again should *not* result in wrapping the existing RPATH wrappers # this can happen when building extensions diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index a334eeca7a..be0cd535c1 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2770,20 +2770,15 @@ def grab_gcc_rpath_wrapper_args(): rpath_wrappers_dir = glob.glob(os.path.join(os.getenv('TMPDIR'), '*', '*', 'rpath_wrappers'))[0] gcc_rpath_wrapper_txt = read_file(glob.glob(os.path.join(rpath_wrappers_dir, '*', 'gcc'))[0]) - # First get the filter argument - rpath_args_regex = re.compile(r"^rpath_args_out=.*rpath_args.py \$CMD '([^ ]*)'.*", re.M) - res_filter = rpath_args_regex.search(gcc_rpath_wrapper_txt) - self.assertTrue(res_filter, "Pattern '%s' found in: %s" % (rpath_args_regex.pattern, gcc_rpath_wrapper_txt)) - - # Now get the include argument - rpath_args_regex = re.compile(r"^rpath_args_out=.*rpath_args.py \$CMD '.*' '([^ ]*)'.*", re.M) - res_include = rpath_args_regex.search(gcc_rpath_wrapper_txt) - self.assertTrue(res_include, "Pattern '%s' found in: %s" % (rpath_args_regex.pattern, - gcc_rpath_wrapper_txt)) + # Get the filter and include arguments + rpath_args_regex = re.compile(r"^readarray -t CMD_ARGS .*rpath_args.py \$CMD " + r"'(?P[^ ]*)' '(?P[^ ]*)'.*", re.M) + res = rpath_args_regex.search(gcc_rpath_wrapper_txt) + self.assertTrue(res, "Pattern '%s' found in: %s" % (rpath_args_regex.pattern, gcc_rpath_wrapper_txt)) shutil.rmtree(rpath_wrappers_dir) - return {'filter_paths': res_filter.group(1), 'include_paths': res_include.group(1)} + return {key: res.group(key) for key in ('filter_paths', 'include_paths')} args = ['--rpath'] self.test_toy_build(extra_args=args, raise_error=True) From eb1144e22e5155cbdf483fefe041cf945f9c2c93 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 4 Sep 2025 10:06:50 +0200 Subject: [PATCH 3/4] Use NULL separator in RPATH script --- easybuild/scripts/rpath_args.py | 2 +- easybuild/scripts/rpath_wrapper_template.sh.in | 2 +- test/framework/toolchain.py | 17 ++++++++++------- test/framework/toy_build.py | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py index 963a17766b..b21a94a93e 100755 --- a/easybuild/scripts/rpath_args.py +++ b/easybuild/scripts/rpath_args.py @@ -183,4 +183,4 @@ def add_rpath_flag(lib_path, rpath_filter, rpath_lib_paths, cmd_args_rpath, ldfl cmd_args = cmd_args_rpath + cmd_args -print('\n'.join(cmd_args)) +print('\0'.join(cmd_args), end='') diff --git a/easybuild/scripts/rpath_wrapper_template.sh.in b/easybuild/scripts/rpath_wrapper_template.sh.in index 63c83f81a0..3ba12429b3 100644 --- a/easybuild/scripts/rpath_wrapper_template.sh.in +++ b/easybuild/scripts/rpath_wrapper_template.sh.in @@ -54,7 +54,7 @@ log "found CMD: $CMD | original command: %(orig_cmd)s | orig args: $*)" # * -S: disable the import of the module site and the site-dependent manipulations of sys.path that it entails; # (once we only support Python 3, we can (also) use -I (isolated mode) log "%(python)s -E -O -s -S %(rpath_args_py)s $CMD '%(rpath_filter)s' '%(rpath_include)s' $*" -readarray -t CMD_ARGS < <( %(python)s -E -O -s -S %(rpath_args_py)s $CMD '%(rpath_filter)s' '%(rpath_include)s' "$@" ) +readarray -d '' -t CMD_ARGS < <( %(python)s -E -O -s -S %(rpath_args_py)s $CMD '%(rpath_filter)s' '%(rpath_include)s' "$@" ) # exclude location of this wrapper from $PATH to avoid other potential wrappers calling this wrapper export PATH=$(echo "$PATH" | tr ':' '\n' | grep -v "^%(wrapper_dir)s$" | tr '\n' ':') diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 6140740ea4..3f90b9f720 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -82,16 +82,19 @@ def tearDown(self): st.get_cpu_vendor = self.orig_get_cpu_vendor super().tearDown() - def run_cmd_and_split(self, cmd): + def run_cmd_and_split(self, cmd, sep='\0'): """Run the command, check return code and return output as list of lines""" with self.mocked_stdout_stderr(): res = run_shell_cmd(cmd) self.assertEqual(res.exit_code, 0) out = res.output - # Remove 1 trailing newline - if out: - self.assertEqual(out[-1], '\n') - return out[:-1].split('\n') + if sep == '\0': + return out.split('\0') + else: + # Remove 1 trailing separator + if out: + self.assertEqual(out[-1], sep) + return out[:-1].split(sep) def get_toolchain(self, name, version=None): """Get a toolchain object instance to test with.""" @@ -3123,7 +3126,7 @@ def test_toolchain_prepare_rpath(self): r"-DY1=\'\'", r"-DY2=\'J\'", ]) - out = self.run_cmd_and_split(cmd) + out = self.run_cmd_and_split(cmd, sep='\n') expected = [ '-Wl,--disable-new-dtags', '-Wl,-rpath=%s/foo' % self.test_prefix, @@ -3165,7 +3168,7 @@ def test_toolchain_prepare_rpath(self): args.append('-L"%s"' % path_with_spaces) cmd = "g++ ${USER}.c %s" % ' '.join(args) - out = self.run_cmd_and_split(cmd) + out = self.run_cmd_and_split(cmd, sep='\n') expected = ('\n'.join([ '-Wl,--disable-new-dtags', diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index d84f356984..032dc41848 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2978,7 +2978,7 @@ def grab_gcc_rpath_wrapper_args(): gcc_rpath_wrapper_txt = read_file(glob.glob(os.path.join(rpath_wrappers_dir, '*', 'gcc'))[0]) # Get the filter and include arguments - rpath_args_regex = re.compile(r"^readarray -t CMD_ARGS .*rpath_args.py \$CMD " + rpath_args_regex = re.compile(r"^readarray -d '' -t CMD_ARGS .*rpath_args.py \$CMD " r"'(?P[^ ]*)' '(?P[^ ]*)'.*", re.M) res = rpath_args_regex.search(gcc_rpath_wrapper_txt) self.assertTrue(res, "Pattern '%s' found in: %s" % (rpath_args_regex.pattern, gcc_rpath_wrapper_txt)) From 976d4be1727f158d617d1762b82799385343b66b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 4 Sep 2025 15:34:58 +0200 Subject: [PATCH 4/4] Use `python -I` for rpath wrapper script Added in Python 3 and implies `-E -s`. So use that fulfilling the TODO. --- easybuild/scripts/rpath_wrapper_template.sh.in | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/easybuild/scripts/rpath_wrapper_template.sh.in b/easybuild/scripts/rpath_wrapper_template.sh.in index 3ba12429b3..f53dff22bc 100644 --- a/easybuild/scripts/rpath_wrapper_template.sh.in +++ b/easybuild/scripts/rpath_wrapper_template.sh.in @@ -48,12 +48,10 @@ log "found CMD: $CMD | original command: %(orig_cmd)s | orig args: $*)" # rpath_args.py script spits out statement that defines $CMD_ARGS # options for 'python' command (see https://docs.python.org/3/using/cmdline.html#miscellaneous-options) -# * -E: ignore all $PYTHON* environment variables that might be set (like $PYTHONPATH); +# * -I: isolated mode: ignore all $PYTHON* environment variables (like $PYTHONPATH) and user site-packages directory # * -O: run Python in optimized mode (remove asserts, ignore stuff under __debug__ guard); -# * -s: don’t add the user site-packages directory to sys.path; # * -S: disable the import of the module site and the site-dependent manipulations of sys.path that it entails; -# (once we only support Python 3, we can (also) use -I (isolated mode) -log "%(python)s -E -O -s -S %(rpath_args_py)s $CMD '%(rpath_filter)s' '%(rpath_include)s' $*" +log "%(python)s -I -O -S %(rpath_args_py)s $CMD '%(rpath_filter)s' '%(rpath_include)s' $*" readarray -d '' -t CMD_ARGS < <( %(python)s -E -O -s -S %(rpath_args_py)s $CMD '%(rpath_filter)s' '%(rpath_include)s' "$@" ) # exclude location of this wrapper from $PATH to avoid other potential wrappers calling this wrapper