From 402a4708f1c4e40d7d78e123c9f93f88202e42b5 Mon Sep 17 00:00:00 2001 From: Lucas Czech Date: Fri, 25 Jul 2025 15:45:21 +0200 Subject: [PATCH 1/3] Refine build tooling for easier local dev --- .github/workflows/main.yml | 22 ++++++++++++++++++++-- build-and-run-tests.py | 3 ++- build.py | 16 ++++++++++++---- test/self-test.py | 15 ++++++++++----- 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8eadb53b..ce8fe3e5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,6 +7,24 @@ on: - cron: "0 0 1/5 * *" jobs: + compilejobUbuntu: + strategy: + fail-fast: false + runs-on: ubuntu-latest + name: Binder_on_Ubuntu + steps: + - name: Install + run: | + set -x + uname -a + cat /etc/issue + sudo apt install python3 python3-dev clang make cmake + - name: Checkout + uses: actions/checkout@v4.0.0 + - name: Compile and Test + run: | + python3 build-and-run-tests.py -j 2 --llvm-version 14.0.5 + compilejobFedora: strategy: fail-fast: false @@ -26,7 +44,7 @@ jobs: libcxx-devel cmake make gcc gcc-c++ \ pybind11-devel python3 python3-devel python2 python2-devel && yum -y clean all - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4.0.0 - name: Compile run: | cmake CMakeLists.txt @@ -41,7 +59,7 @@ jobs: name: Binder_on_OSX steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4.0.0 - name: Install run: | set -x diff --git a/build-and-run-tests.py b/build-and-run-tests.py index f415169c..05780edb 100755 --- a/build-and-run-tests.py +++ b/build-and-run-tests.py @@ -37,13 +37,14 @@ def main(args): parser.add_argument('--annotate-includes', action="store_true", help='Annotate includes in generated source files') parser.add_argument('--trace', action="store_true", help='Binder will add trace output to to generated source files') parser.add_argument('--gcc-install-prefix', default=None, help='Path to GCC install prefix which will be used to determent location of libstdc++ for Binder build. Default is: auto-detected. Use this option if you would like to build Binder with compiler that was side-installed and which LLVM build system failed to identify. To see what path Binder uses for libstdc++ run `binder -- -xc++ -E -v`.') + parser.add_argument('--update-binder', action="store_true", help='Recompile the Binder binary; useful when working on the Binder code') global Options Options = parser.parse_args() source_path = os.path.abspath('.') - if not Options.binder: Options.binder = build.install_llvm_tool('binder', source_path+'/source', source_path + '/build', Options.binder_debug, jobs=Options.jobs, gcc_install_prefix=Options.gcc_install_prefix, compiler=Options.compiler, llvm_version=Options.llvm_version) + if not Options.binder: Options.binder = build.install_llvm_tool('binder', source_path+'/source', source_path + '/build', Options.binder_debug, jobs=Options.jobs, gcc_install_prefix=Options.gcc_install_prefix, compiler=Options.compiler, llvm_version=Options.llvm_version, update_binder=Options.update_binder) if not Options.pybind11: Options.pybind11 = build.install_pybind11(source_path + '/build') diff --git a/build.py b/build.py index 49714379..a97ebe4b 100644 --- a/build.py +++ b/build.py @@ -29,8 +29,8 @@ _python_version_ = '{}.{}'.format(sys.version_info.major, sys.version_info.minor) # should be formatted: 2.7, 3.5, 3.6, ... -_pybind11_version_ = 'aa304c9c7d725ffb9d10af08a3b34cb372307020' - +#_pybind11_version_ = 'aa304c9c7d725ffb9d10af08a3b34cb372307020' +_pybind11_version_ = 'a2e59f0e7065404b44dfe92a28aca47ba1378dc4' def execute(message, command_line, return_='status', until_successes=False, terminate_on_failure=True, silent=False, silence_output=False): if not silent: print(message); print(command_line); sys.stdout.flush(); @@ -89,7 +89,7 @@ def get_cmake_compiler_options(compiler): return '' -def install_llvm_tool(name, source_location, prefix_root, debug, compiler, jobs, gcc_install_prefix, clean=True, llvm_version=None): +def install_llvm_tool(name, source_location, prefix_root, debug, compiler, jobs, gcc_install_prefix, clean=True, llvm_version=None, update_binder=False): ''' Install and update (if needed) custom LLVM tool at given prefix (from config). Return absolute path to executable on success and terminate with error on failure ''' @@ -219,6 +219,13 @@ def install_llvm_tool(name, source_location, prefix_root, debug, compiler, jobs, if not os.path.isfile(cmake_lists): with open(cmake_lists, 'w') as f: f.write(tool_build_line + '\n') + # We are building from scratch, so we want to build Binder as well. + update_binder = True + + # The above takes care of LLVM being built and ready. However, we might want to update just + # the build for Binder itself, without having to recompile all of LLVM, for instance during dev. + # Thus, here, we can re-run cmake to just build the binder binary again. + if update_binder: config = '-DCMAKE_BUILD_TYPE={build_type}'.format(build_type='Debug' if debug else 'Release') config += get_cmake_compiler_options(compiler) @@ -278,13 +285,14 @@ def main(args): parser.add_argument('--llvm-version', default=None, choices=['6.0.1', '13.0.0', '14.0.5'], help='Manually specify LLVM version to install') parser.add_argument('--annotate-includes', action="store_true", help='Annotate includes in generated source files') parser.add_argument('--trace', action="store_true", help='Binder will add trace output to to generated source files') + parser.add_argument('--update-binder', action="store_true", help='Recompile the Binder binary; useful when working on the Binder code') global Options Options = parser.parse_args() source_path = os.path.abspath('.') - if not Options.binder: Options.binder = install_llvm_tool('binder', source_path+'/source', source_path + '/build', debug=Options.binder_debug, compiler=Options.compiler, jobs=Options.jobs, gcc_install_prefix=None, llvm_version=Options.llvm_version) + if not Options.binder: Options.binder = install_llvm_tool('binder', source_path+'/source', source_path + '/build', debug=Options.binder_debug, compiler=Options.compiler, jobs=Options.jobs, gcc_install_prefix=None, llvm_version=Options.llvm_version, update_binder=Options.update_binder) if not Options.pybind11: Options.pybind11 = install_pybind11(source_path + '/build') diff --git a/test/self-test.py b/test/self-test.py index e6dc7c63..004d9bf5 100755 --- a/test/self-test.py +++ b/test/self-test.py @@ -131,14 +131,17 @@ def run_test(test_path, build_dir, pyenv): command_line = 'diff {ref} {new}'.format(**vars()) remover_absolute_paths(new) - r = execute('Comparing results for test {}...'.format(test), command_line, terminate_on_failure=not Options.accept); + r = execute('Comparing results for test {}...'.format(test), command_line, terminate_on_failure=not (Options.accept or Options.accept_all )); not_binded = [l for l in open(new).read().split('\n') if 'not_binded' in l] if not_binded: print('{}\n"not_binded" string found in results for test {}!!!\n'.format('\n'.join(not_binded), test)); sys.exit(1) - if r and Options.accept: - a = input( 'Accept new results from test {test} as reference? [Y/n] '.format(test=test) ) - if a in ['', 'y', 'yes']: shutil.copyfile(new, ref) + if r: + if Options.accept and not Options.accept_all: + a = input( 'Accept new results from test {test} as reference? [Y/n] '.format(test=test) ) + if a in ['', 'y', 'yes']: shutil.copyfile(new, ref) + if Options.accept_all: + shutil.copyfile(new, ref) @@ -151,7 +154,8 @@ def main(): parser.add_argument('--pybind11', default='', help='Path to pybind11 source tree') - parser.add_argument("--accept", action="store_true", help="Run tests and accept new tests results as reference") + parser.add_argument("--accept", action="store_true", help="Run tests and ask to accept new tests results as reference") + parser.add_argument("--accept-all", action="store_true", help="Run tests and automatically accept all new tests results as reference") parser.add_argument("--annotate", action="store_true", help="Run Binder with extra annotation options") parser.add_argument('args', nargs=argparse.REMAINDER, help='Optional: list of tests to run') @@ -162,6 +166,7 @@ def main(): test_source_dir = os.path.dirname( os.path.abspath(__file__) ) tests = Options.args or [ t for t in sorted( get_test_files(test_source_dir) ) if 'T61.smart_holder.hpp' not in t ] + print('Found {ntests} test cases'.format( ntests=len(tests) )) build_dir = test_source_dir + '/build' if os.path.isdir(build_dir): print('Removing old test dir {0} ...'.format(build_dir)); shutil.rmtree(build_dir) # remove old dir if any From 270ac2c73a8fb2a7d317660780c9ec2b617d7064 Mon Sep 17 00:00:00 2001 From: Lucas Czech Date: Fri, 25 Jul 2025 15:45:57 +0200 Subject: [PATCH 2/3] Fix missing header include in test case --- test/T42.stl.names.multi.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/test/T42.stl.names.multi.hpp b/test/T42.stl.names.multi.hpp index a4197f0a..f00db25c 100644 --- a/test/T42.stl.names.multi.hpp +++ b/test/T42.stl.names.multi.hpp @@ -26,6 +26,7 @@ #include #include #include +#include #include void foo(std::multimap) From 72bf15d1ead88c71fcdc171668e5941357023bd4 Mon Sep 17 00:00:00 2001 From: Lucas Czech Date: Sat, 26 Jul 2025 14:16:54 +0200 Subject: [PATCH 3/3] Refine build.py for incremental Binder builds --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index a97ebe4b..8436074e 100644 --- a/build.py +++ b/build.py @@ -154,7 +154,7 @@ def install_llvm_tool(name, source_location, prefix_root, debug, compiler, jobs, if signature == disk_signature: print('LLVM:{} + Binder install is detected at {}, skipping LLVM installation and Binder building procedures...\n'.format(llvm_version, build_dir)) - else: + elif not update_binder: print('LLVM build detected, but config/binder version has changed, perfoming a clean rebuild...') if os.path.isdir(build_dir): shutil.rmtree(build_dir) @@ -285,7 +285,7 @@ def main(args): parser.add_argument('--llvm-version', default=None, choices=['6.0.1', '13.0.0', '14.0.5'], help='Manually specify LLVM version to install') parser.add_argument('--annotate-includes', action="store_true", help='Annotate includes in generated source files') parser.add_argument('--trace', action="store_true", help='Binder will add trace output to to generated source files') - parser.add_argument('--update-binder', action="store_true", help='Recompile the Binder binary; useful when working on the Binder code') + parser.add_argument('--update-binder', action="store_true", help='Recompile only the Binder binary; useful when working on the Binder code') global Options Options = parser.parse_args()