From 2240095af6333e5f1823f1d925d9578275c457b6 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Tue, 7 Oct 2025 03:04:37 -0700 Subject: [PATCH 1/7] Initial step towards python proc visibility --- CMakeLists.txt | 7 +++---- Testing/Cpp/TestProcessorVisibility.cc | 15 ++++++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 278befa..7e95a28 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,23 +76,22 @@ list( APPEND PUBLIC_EXT_LIBS Boost::boost Boost::chrono Boost::date_time Boost:: set( Boost_USE_STATIC_LIBS OFF ) set( Boost_USE_STATIC_RUNTIME OFF ) set( Boost_ALL_DYN_LINK ON ) -add_definitions( -DBOOST_ALL_DYN_LINK ) +add_compile_definitions( -DBOOST_ALL_DYN_LINK ) ########## # Pybind11 ########## if( Nymph_ENABLE_PYTHON ) - find_package( Python3 REQUIRED COMPONENTS Interpreter Development ) set(PYBIND11_PYTHON_VERSION ${Python3_VERSION} CACHE STRING "") find_package( pybind11 REQUIRED ) include_directories( ${Python3_INCLUDE_DIRS} ) - + add_compile_definitions( NYMPH_USING_PYTHON ) set_option( Scarab_BUILD_PYTHON TRUE ) add_subdirectory( Python/Bindings ) - else( Nymph_ENABLE_PYTHON ) + remove_definitions( NYMPH_USING_PYTHON ) set_option( Scarab_BUILD_PYTHON FALSE ) endif( Nymph_ENABLE_PYTHON ) diff --git a/Testing/Cpp/TestProcessorVisibility.cc b/Testing/Cpp/TestProcessorVisibility.cc index 45e7051..0fcbd7d 100644 --- a/Testing/Cpp/TestProcessorVisibility.cc +++ b/Testing/Cpp/TestProcessorVisibility.cc @@ -32,10 +32,15 @@ TEST_CASE( "processor_visibility" ) } -// SECTION( "Python" ) -// { -// LINFO( testlog, "Python Processor Visibility"); -// -// } + SECTION( "Python" ) + { + LINFO( testlog, "Python Processor Visibility"); + +#ifdef NYMPH_USING_PYTHON + REQUIRE( tptToolbox.CouldBuild( "hello-world-python" ) ); +#else + REQUIRE_FALSE( tptToolbox.CouldBuild( "hello-world-python" ) ); +#endif + } } From e8139ce3aefb55a298e2848444b56683ba5e0df3 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 8 Oct 2025 14:26:00 -0700 Subject: [PATCH 2/7] Python import testing --- Cpp/Executables/Main/CMakeLists.txt | 12 ++++++++++++ Cpp/Executables/Main/PyImportTest.cc | 12 ++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 Cpp/Executables/Main/PyImportTest.cc diff --git a/Cpp/Executables/Main/CMakeLists.txt b/Cpp/Executables/Main/CMakeLists.txt index 4bfc255..c7ad842 100644 --- a/Cpp/Executables/Main/CMakeLists.txt +++ b/Cpp/Executables/Main/CMakeLists.txt @@ -17,6 +17,18 @@ if( Nymph_BUILD_NYMPH_EXE ) ) endif( Nymph_BUILD_NYMPH_EXE ) +if( Nymph_ENABLE_PYTHON ) + set( Nymph_SOURCES + ${Nymph_SOURCES} + PyImportTest.cc + ) + set( Nymph_EXE_LIBRARIES + ${Nymph_EXE_LIBRARIES} + pybind11::embed + pybind11::module + ) +endif() + if( Nymph_SOURCES ) set( Nymph_PROGRAMS ) pbuilder_executables( diff --git a/Cpp/Executables/Main/PyImportTest.cc b/Cpp/Executables/Main/PyImportTest.cc new file mode 100644 index 0000000..83bf0da --- /dev/null +++ b/Cpp/Executables/Main/PyImportTest.cc @@ -0,0 +1,12 @@ + +#include + +namespace py = pybind11; + +int main() +{ + py::module_ sys = py::module_::import("sys"); + py::print(sys.attr("path")); + + return 0; +} From b0771c60f8b98c87fb57f0e4c418278b60ec2fdf Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 8 Oct 2025 14:26:27 -0700 Subject: [PATCH 3/7] Preliminary work testing python proc visibility in C++ --- Testing/CMakeLists.txt | 9 ++++++- Testing/Cpp/TestProcessorVisibility.cc | 37 +++++++++++++++++++------- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/Testing/CMakeLists.txt b/Testing/CMakeLists.txt index 7f477bc..74f2f57 100644 --- a/Testing/CMakeLists.txt +++ b/Testing/CMakeLists.txt @@ -90,6 +90,13 @@ set( testing_LIB_DEPENDENCIES NymphTesting ) +#if( Nymph_ENABLE_PYTHON ) +# set( testing_LIB_DEPENDENCIES +# ${testing_LIB_DEPENDENCIES} +# nymph_bindings +# ) +#endif( Nymph_ENABLE_PYTHON ) + # Interface library was recommended for Catch: # https://stackoverflow.com/questions/34896891/catch-lib-unit-testing-and-ctest-cmake-integration/34900012#34900012 #add_library( Catch INTERFACE ) @@ -99,7 +106,7 @@ pbuilder_executable( EXECUTABLE RunTests SOURCES ${testing_SOURCES} PROJECT_LIBRARIES ${testing_LIB_DEPENDENCIES} - PUBLIC_EXTERNAL_LIBRARIES Catch2::Catch2 + PUBLIC_EXTERNAL_LIBRARIES Catch2::Catch2 pybind11::module pybind11::embed ) set( programs "RunTests" ) diff --git a/Testing/Cpp/TestProcessorVisibility.cc b/Testing/Cpp/TestProcessorVisibility.cc index 0fcbd7d..8cd2477 100644 --- a/Testing/Cpp/TestProcessorVisibility.cc +++ b/Testing/Cpp/TestProcessorVisibility.cc @@ -11,8 +11,12 @@ #include "logger.hh" +#include + #include "catch2/catch_test_macros.hpp" +namespace py = pybind11; + LOGGER( testlog, "TestProcessorVisibility" ); TEST_CASE( "processor_visibility" ) @@ -32,15 +36,28 @@ TEST_CASE( "processor_visibility" ) } - SECTION( "Python" ) - { - LINFO( testlog, "Python Processor Visibility"); - -#ifdef NYMPH_USING_PYTHON - REQUIRE( tptToolbox.CouldBuild( "hello-world-python" ) ); -#else - REQUIRE_FALSE( tptToolbox.CouldBuild( "hello-world-python" ) ); -#endif - } +// SECTION( "Python - no import" ) +// { +// LINFO( testlog, "Python Processor Visibility"); +// +//#ifdef NYMPH_USING_PYTHON +// REQUIRE_FALSE( tptToolbox.CouldBuild( "hello-world-python" ) ); +//#else +// REQUIRE_FALSE( tptToolbox.CouldBuild( "hello-world-python" ) ); +//#endif +// } +// +// SECTION( "Python - with import" ) +// { +// LINFO( testlog, "Python Processor Visibility"); +// +//#ifdef NYMPH_USING_PYTHON +// py::module_ nymph = py::module_::import("nymph"); +// +// REQUIRE( tptToolbox.CouldBuild( "hello-world-python" ) ); +//#else +// REQUIRE_FALSE( tptToolbox.CouldBuild( "hello-world-python" ) ); +//#endif +// } } From d2deecb9977ef4ff0fd527b9423661fe60aacd89 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 8 Oct 2025 15:53:39 -0700 Subject: [PATCH 4/7] Reverting a small change to a boost flag --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e95a28..08f9633 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,7 +76,7 @@ list( APPEND PUBLIC_EXT_LIBS Boost::boost Boost::chrono Boost::date_time Boost:: set( Boost_USE_STATIC_LIBS OFF ) set( Boost_USE_STATIC_RUNTIME OFF ) set( Boost_ALL_DYN_LINK ON ) -add_compile_definitions( -DBOOST_ALL_DYN_LINK ) +add_definitions( -DBOOST_ALL_DYN_LINK ) ########## # Pybind11 From 377a51417568d41b6c82d8b424a32d6304277a83 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 8 Oct 2025 16:02:26 -0700 Subject: [PATCH 5/7] Fixed the import test --- Cpp/Executables/Main/PyImportTest.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Cpp/Executables/Main/PyImportTest.cc b/Cpp/Executables/Main/PyImportTest.cc index 83bf0da..e52669e 100644 --- a/Cpp/Executables/Main/PyImportTest.cc +++ b/Cpp/Executables/Main/PyImportTest.cc @@ -1,10 +1,13 @@ #include +#include namespace py = pybind11; int main() { + py::scoped_interpreter guard{}; + py::module_ sys = py::module_::import("sys"); py::print(sys.attr("path")); From efd421fe35e24df4d6af049e1e63f9394ad7845b Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 8 Oct 2025 16:50:37 -0700 Subject: [PATCH 6/7] Improved processor visibility test for python processors from C++ --- Testing/CMakeLists.txt | 16 ++++---- Testing/Cpp/TestProcessorVisibility.cc | 57 +++++++++++++++----------- 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/Testing/CMakeLists.txt b/Testing/CMakeLists.txt index 74f2f57..a2c268d 100644 --- a/Testing/CMakeLists.txt +++ b/Testing/CMakeLists.txt @@ -90,12 +90,14 @@ set( testing_LIB_DEPENDENCIES NymphTesting ) -#if( Nymph_ENABLE_PYTHON ) -# set( testing_LIB_DEPENDENCIES -# ${testing_LIB_DEPENDENCIES} -# nymph_bindings -# ) -#endif( Nymph_ENABLE_PYTHON ) +set( testing_PUBLIC_LIBS Catch2::Catch2 ) + +if( Nymph_ENABLE_PYTHON ) + set( testing_PUBLIC_LIBS + ${testing_PUBLIC_LIBS} + pybind11::embed + ) +endif( Nymph_ENABLE_PYTHON ) # Interface library was recommended for Catch: # https://stackoverflow.com/questions/34896891/catch-lib-unit-testing-and-ctest-cmake-integration/34900012#34900012 @@ -106,7 +108,7 @@ pbuilder_executable( EXECUTABLE RunTests SOURCES ${testing_SOURCES} PROJECT_LIBRARIES ${testing_LIB_DEPENDENCIES} - PUBLIC_EXTERNAL_LIBRARIES Catch2::Catch2 pybind11::module pybind11::embed + PUBLIC_EXTERNAL_LIBRARIES ${testing_PUBLIC_LIBS} ) set( programs "RunTests" ) diff --git a/Testing/Cpp/TestProcessorVisibility.cc b/Testing/Cpp/TestProcessorVisibility.cc index 8cd2477..4a3ac5c 100644 --- a/Testing/Cpp/TestProcessorVisibility.cc +++ b/Testing/Cpp/TestProcessorVisibility.cc @@ -11,11 +11,12 @@ #include "logger.hh" -#include - #include "catch2/catch_test_macros.hpp" +#ifdef NYMPH_USING_PYTHON +#include namespace py = pybind11; +#endif LOGGER( testlog, "TestProcessorVisibility" ); @@ -36,28 +37,34 @@ TEST_CASE( "processor_visibility" ) } -// SECTION( "Python - no import" ) -// { -// LINFO( testlog, "Python Processor Visibility"); -// -//#ifdef NYMPH_USING_PYTHON -// REQUIRE_FALSE( tptToolbox.CouldBuild( "hello-world-python" ) ); -//#else -// REQUIRE_FALSE( tptToolbox.CouldBuild( "hello-world-python" ) ); -//#endif -// } -// -// SECTION( "Python - with import" ) -// { -// LINFO( testlog, "Python Processor Visibility"); -// -//#ifdef NYMPH_USING_PYTHON -// py::module_ nymph = py::module_::import("nymph"); -// -// REQUIRE( tptToolbox.CouldBuild( "hello-world-python" ) ); -//#else -// REQUIRE_FALSE( tptToolbox.CouldBuild( "hello-world-python" ) ); -//#endif -// } + SECTION( "Python - no import" ) + { + LINFO( testlog, "Python Processor Visibility (no python import)"); + + REQUIRE_FALSE( tptToolbox.CouldBuild( "hello-world-python" ) ); + } + +#ifdef NYMPH_USING_PYTHON + SECTION( "Python - with import" ) + { + py::scoped_interpreter guard{}; +// py::module_ sys = py::module_::import("sys"); +// py::print(sys.attr("path")); + + LINFO( testlog, "Python Processor Visibility (with python import)"); + + try + { + py::module_ nymph = py::module_::import("nymph"); + REQUIRE( tptToolbox.CouldBuild( "hello-world-py" ) ); + } + catch(const std::exception& e) + { + LWARN( testlog, "Unable to import `nymph` ") + } + + + } +#endif } From 003686f9b65c80bae8515f44079b28d361e9fae4 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 9 Oct 2025 09:32:10 -0700 Subject: [PATCH 7/7] Fixed up calls within nymph_bindings tests --- Testing/Python/Bindings/testprocessor.py | 4 ++-- Testing/Python/Bindings/testprocfactory.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Testing/Python/Bindings/testprocessor.py b/Testing/Python/Bindings/testprocessor.py index d8019ca..2829c55 100644 --- a/Testing/Python/Bindings/testprocessor.py +++ b/Testing/Python/Bindings/testprocessor.py @@ -60,8 +60,8 @@ class TestPyProcCreator(unittest.TestCase): def test_creating(self): - registrar = nymph_bindings.processor.register_py_processor('testprocessor', 'TestProcessor', 'processor-name') - test_proc = nymph_bindings.processor.create_processor('processor-name', 'test-proc') + registrar = nymph_bindings.processor._register('testprocessor', 'TestProcessor', 'processor-name') + test_proc = nymph_bindings.processor._create('processor-name', 'test-proc') self.assertEqual(test_proc.name, 'test-proc') diff --git a/Testing/Python/Bindings/testprocfactory.py b/Testing/Python/Bindings/testprocfactory.py index 5a49e2b..b809d99 100644 --- a/Testing/Python/Bindings/testprocfactory.py +++ b/Testing/Python/Bindings/testprocfactory.py @@ -16,7 +16,7 @@ class TestNoProc(unittest.TestCase): '''Testing the situation where the processor type doesn't exist''' def test_noproc(self): with self.assertRaises(RuntimeError) as cm: - nb.processor.create( "blah", "blah-blah" ) + nb.processor._create( "blah", "blah-blah" ) the_exception = cm.exception self.assertEqual(str(the_exception), "Did not find processor with type ") @@ -24,7 +24,7 @@ class TestCppProc(unittest.TestCase): '''Testing the creation of a C++ processor, which won't work''' def test_cppproc(self): with self.assertRaises(RuntimeError) as cm: - nb.processor.create( "hello-world-cpp", "hw" ) + nb.processor._create( "hello-world-cpp", "hw" ) the_exception = cm.exception self.assertEqual(str(the_exception), "Registrar did not cast correctly for ")