Pycv package ci #44

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft

Iximiel wants to merge 24 commits into master from pycv-package-CI

.github/workflows/ci.yml

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -19,18 +19,18 @@ jobs:
  
        uses: ./.github/workflows/linuxWF.yml

        secrets: inherit

      codecheck:

        uses: ./.github/workflows/codecheckWF.yml

        secrets: inherit

      # codecheck:

      #   uses: ./.github/workflows/codecheckWF.yml

      #   secrets: inherit

      mac:

        uses: ./.github/workflows/macWF.yml

        secrets: inherit

      # mac:

      #   uses: ./.github/workflows/macWF.yml

      #   secrets: inherit

      docker:

        uses: ./.github/workflows/dockerWF.yml

        secrets: inherit

      # docker:

      #   uses: ./.github/workflows/dockerWF.yml

      #   secrets: inherit

      conda:

        uses: ./.github/workflows/condaWF.yml

        secrets: inherit

      # conda:

      #   uses: ./.github/workflows/condaWF.yml

      #   secrets: inherit

.github/workflows/linuxWF.yml

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -19,22 +19,21 @@ jobs:
  
          fail-fast: false

          matrix:

            variant:

            - -doc-mpi-

            - -mpi-

            - -coverage-mpi-

            - -debug-

            - -debug-mpi-

            # - -doc-mpi-

            # - -mpi-

            # - -coverage-mpi-

            # - -debug-

            # - -debug-mpi-

    # temporarily commented out

    # see https://github.com/plumed/plumed2/issues/976

            - -intel-

            # - -intel-

            - -pycv-mpi-

        steps:

        - uses: actions/checkout@v4

        - uses: actions/cache@v4

        - uses: actions/cache/restore@v4

          with:

            path: ~/.ccache

            key: ccache-reset1-linux${{ matrix.variant }}hash-${{ github.sha }}

            restore-keys: ccache-reset1-linux${{ matrix.variant }}hash-

            key: ccache-reset1-linux${{ matrix.variant }}

        - name: Removed unused stuff

          run: |

            df -h

    @@ -159,16 +158,16 @@ jobs:
  
            # check for global symbols, see https://github.com/plumed/plumed2/issues/549

            make nmcheck

            ccache -s -M 100M

        - name: Run tests

          if: ${{ ! contains( matrix.variant, '-doc-mpi-' ) && ! contains( matrix.variant, '-pycv-mpi-' ) }}

          run: |

             (while true; do  # see https://github.com/actions/virtual-environments/issues/1860

                  df -h

                  sleep 15

                  done) &

             make --no-print-directory -C regtest testclean

             # these can fail for numerical reasons

             make -C regtest checkfail

        # - name: Run tests

        #   if: ${{ ! contains( matrix.variant, '-doc-mpi-' ) && ! contains( matrix.variant, '-pycv-mpi-' ) }}

        #   run: |

        #      (while true; do  # see https://github.com/actions/virtual-environments/issues/1860

        #           df -h

        #           sleep 15

        #           done) &

        #      make --no-print-directory -C regtest testclean

        #      # these can fail for numerical reasons

        #      make -C regtest checkfail

        - name: Run python tests

          run: |

             cd python

    @@ -194,12 +193,15 @@ jobs:
  
            GIT_TOKEN: ${{ secrets.GIT_TOKEN_PLUMEDBOT }}

          run: |

             .ci/push doc

        - uses: actions/cache/save@v4

          with:

               path: ~/.ccache

               key: ccache-reset1-linux${{ matrix.variant }}

        - name: Compile and test pycv

          if: contains( matrix.variant, '-pycv-' )

          working-directory: ./plugins/pycv/

          run: |

                pip install --user -r requirements.txt

                source ../../sourceme.sh

                ln -s $(realpath ../../regtest/scripts) ./regtest/scripts

                ./prepareMakeForDevelop.sh

                make check

plugins/pycv/.gitignore

-Original file line number
+Diff line change
@@ -1,4 +1,7 @@
     /Makefile.conf
     *.o
     *.so
-    env
+    /env*/
+    /build*/
+    /dist*/
+    *.whl

plugins/pycv/CMakeLists.txt

-Original file line number
+Diff line change
@@ -0,0 +1,75 @@
+    cmake_minimum_required(VERSION 3.15...3.27)
+    project(
+      ${SKBUILD_PROJECT_NAME}
+      VERSION ${SKBUILD_PROJECT_VERSION}
+      LANGUAGES CXX)
+    set(CMAKE_CXX_STANDARD 17)
+    message(
+      STATUS
+        "Everithing should work fine if you are in the same environment in which you have compiled plumed"
+    )
+    # FinPlumed is here:
+    list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}")
+    # Finding necessary packages
+    find_package(Python REQUIRED COMPONENTS Interpreter Development)
+    find_package(pybind11 CONFIG REQUIRED)
+    find_package(Plumed REQUIRED)
+    # Finding optionals things
+    if(Plumed_HAS_MPI)
+      find_package(MPI REQUIRED)
+    endif()
+    if(MPI_CXX_FOUND)
+      list(APPEND extraLibs MPI::MPI_CXX)
+    endif()
+    if(Plumed_HAS_OPENMP)
+      find_package(OpenMP REQUIRED)
+    endif()
+    if(OpenMP_CXX_FOUND)
+      list(APPEND extraLibs OpenMP::OpenMP_CXX)
+    endif()
+    include(CheckCXXCompilerFlag)
+    check_cxx_compiler_flag(-fno-gnu-unique USE_NO_GNU_UNIQUE)
+    if(USE_NO_GNU_UNIQUE)
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-gnu-unique")
+    endif()
+    # plumed_STATIC_LDFLAGS_OTHER:INTERNAL=-rdynamic;-Wl,-Bsymbolic;-fopenmp
+    # -rdynamic is automatically set by cmake, and also -fPIC
+    ################################################################################
+    ################################the pycv library################################
+    ################################################################################
+    add_library(
+      PythonCVInterface SHARED src/ActionWithPython.cpp src/PythonCVInterface.cpp
+                               src/PythonFunction.cpp)
+    # public, so they trickle down to the python module
+    target_compile_definitions(PythonCVInterface PUBLIC ${Plumed_CFLAGS})
+    target_include_directories(PythonCVInterface PUBLIC src ${Plumed_INCLUDEDIR})
+    # ####################################################################
+    # uncommenting this brings problems since some symbols here are needed
+    # by the python module even if it should be the correct setting...
+    # https://gcc.gnu.org/wiki/Visibility could be a starting point
+    #######################################################################
+    # target_compile_options(PythonCVInterface PRIVATE -fvisibility=hidden)
+    target_link_libraries(PythonCVInterface PRIVATE pybind11::embed)
+    target_link_libraries(PythonCVInterface PUBLIC plumedKernel ${extraLibs})
+    # this removes the "lib" prefix
+    set_target_properties(PythonCVInterface PROPERTIES PREFIX "")
+    install(TARGETS PythonCVInterface DESTINATION pycv)
+    ################################################################################
+    ###########################The pvCV companion module############################
+    ################################################################################
+    pybind11_add_module(plumedCommunications src/PlumedPythonEmbeddedModule.cpp)
+    target_link_libraries(plumedCommunications PRIVATE pybind11::headers)
+    target_link_libraries(plumedCommunications PUBLIC PythonCVInterface)
+    # The install directory is the output (wheel) directory
+    install(TARGETS plumedCommunications DESTINATION .)

plugins/pycv/FindPlumed.cmake

-Original file line number
+Diff line change
@@ -0,0 +1,87 @@
+    if(NOT Plumed_FOUND)
+      find_package(PkgConfig)
+      if(Plumed_FIND_QUIETLY)
+        function(message)
+          # THIS completely shuts down messages
+        endfunction()
+        pkg_check_modules(PLUMED QUIET plumedInternals)
+      else()
+        pkg_check_modules(PLUMED plumedInternals)
+      endif()
+      if(Plumed_FOUND)
+        if("-D__PLUMED_HAS_MPI=1" IN_LIST Plumed_CFLAGS)
+          set(Plumed_HAS_MPI
+              CACHE INTERNAL "plumed has MPI")
+        endif()
+        if("-fopenmp" IN_LIST Plumed_STATIC_LDFLAGS_OTHER)
+          set(Plumed_HAS_OPENMP
+              CACHE INTERNAL "plumed has OpenMP")
+        endif()
+      else()
+        message(STATUS "plumed not found via pkgconfig, trying executable")
+        execute_process(
+          COMMAND plumed info --include-dir
+          RESULT_VARIABLE PLUMED_EXECUTABLE
+          OUTPUT_QUIET ERROR_QUIET)
+        if(PLUMED_EXECUTABLE EQUAL 0)
+          set(Plumed_FOUND
+              CACHE INTERNAL "plumed found")
+          message(STATUS "Configuring plumed from executable")
+          execute_process(
+            COMMAND plumed info --include-dir
+            OUTPUT_VARIABLE Plumed_INCLUDEDIR
+            OUTPUT_STRIP_TRAILING_WHITESPACE)
+          set(Plumed_INCLUDEDIR
+              ${Plumed_INCLUDEDIR}
+              CACHE INTERNAL "plumed include dir")
+          execute_process(
+            COMMAND plumed info --configuration
+            OUTPUT_VARIABLE Plumed_CONFIG
+            OUTPUT_STRIP_TRAILING_WHITESPACE)
+          set(Plumed_CPP_FLAGS "")
+          string(REPLACE "\n" ";" ProcessFile_LINES "${Plumed_CONFIG}")
+          foreach(_line ${ProcessFile_LINES})
+            if(${_line} MATCHES "CPPFLAGS=.*")
+              set(Plumed_CPP_FLAGS ${_line})
+              string(REGEX REPLACE "CPPFLAGS= *" "" Plumed_CPP_FLAGS
+                                   ${Plumed_CPP_FLAGS})
+              string(REPLACE "\\" "" Plumed_CPP_FLAGS ${Plumed_CPP_FLAGS})
+              string(REPLACE "-D" ";" Plumed_CPP_FLAGS ${Plumed_CPP_FLAGS})
+              # message(STATUS "Found PLUMED CPP_FLAGS: \"${Plumed_CPP_FLAGS}\"")
+              # message(STATUS "Found PLUMED CPP_FLAGS:") foreach(_flag
+              # ${Plumed_CPP_FLAGS}) message(STATUS "   \"${_flag}\"") endforeach()
+            endif()
+            if(${_line} MATCHES ".*-fopenmp.*")
+              set(Plumed_HAS_MPI
+                  CACHE INTERNAL "plumed has MPI")
+            endif()
+          endforeach()
+          set(Plumed_CFLAGS
+              ${Plumed_CPP_FLAGS}
+              CACHE INTERNAL "plumed Definitions flags")
+          execute_process(COMMAND plumed config -q has mpi
+                          RESULT_VARIABLE Plumed_WITH_MPI)
+          if(Plumed_WITH_MPI EQUAL 0)
+            set(Plumed_HAS_MPI
+                CACHE INTERNAL "plumed has MPI")
+          endif()
+        else()
+          if(Plumed_FIND_REQUIRED)
+            message(FATAL_ERROR "plumed not found")
+          endif()
+        endif()
+      endif()
+    endif()

plugins/pycv/Makefile

-Original file line number
+Diff line change
@@ -1,53 +1,24 @@
-    # include the machine dependent configuration
-    ifneq ($(MAKECMDGOALS),clean)
-      -include ../../Makefile.conf
-    	ifndef canPyCV
-      	include ./Makefile.conf
-    	endif
-    endif
+    #this makefiles assume that pip and pytest are installed
+    .PHONY: clean check check_standalone check_python all
-    .PHONY: clean check all
-    #Dependency tracking based on https://make.mad-scientist.net/papers/advanced-auto-dependency-generation/#tldr
-    #this assumes gcc
-    DEPDIR := .deps
-    DEPFLAGS = -MT $@ -MMD -MP -MF $(DEPDIR)/$*.d
-    PYBIND11FLAG =
-    ADDCPPFLAGS:=$(python_cf_embedded) $(pybind11_cflags) $(ADDCPPFLAGS)
-    ADDCLDFLAGS:=$(ADDCLDFLAGS) $(python_ld_embedded)
-    OBJS = ActionWithPython.o PythonCVInterface.o PythonFunction.o PlumedPythonEmbeddedModule.o
+    all: pycv_here
-    ifeq ($(SOEXT),dylib)
-      SONAME_OPTION:=-Wl,-install_name
-    else
-      SONAME_OPTION:=-Wl,-soname
-    endif
-    all: PythonCVInterface.$(SOEXT)
-    #-fvisibility=hidden is needed for pybind11 (to not conflict with different pybind11 versions)
-    #I think I enforced this nearly everywhere I set up a flag for the compiler
-    ActionWithPython.o PythonCVInterface.o PythonFunction.o PlumedPythonEmbeddedModule.o: PYBIND11FLAG= -fvisibility=hidden
-    %.o: %.cpp $(DEPDIR)/%.d | $(DEPDIR)
-    	@echo Compiling object $@
-    	@$(CXX) -c $(DEPFLAGS) $(CPPFLAGS) $(PYBIND11FLAG) $(ADDCPPFLAGS) $(CXXFLAGS) $< -o $@
-    $(DEPDIR): ; @mkdir -p $@
-    DEPFILES := $(OBJS:%.o=$(DEPDIR)/%.d)
-    $(DEPFILES):
-    include $(wildcard $(DEPFILES))
-    #-Wl,--no-as-needed forces the python library to be linked, without this in a WSL does not work
-    #TODO: seems that $PLUMED_KERNEL is not needed, check
-    PythonCVInterface.$(SOEXT): $(OBJS)
-    	@echo Linking $@
-    	$(LDSHARED) $(SONAME_OPTION),"$(notdir $@)" $(DYNAMIC_LIBS) $(PLUMED_KERNEL) $(ADDCLDFLAGS) $^ -o $@
+    pycv_here: src/*.cpp src/*.h src/pycv/*.py
+    	@echo installing pycv
+    	pip install .
+    	touch $@
     clean:
-    	rm -f $(OBJS) PythonCVInterface.$(SOEXT)
+    	pip uninstall pycv
+    	rm -fv pycv_here
-    check: all
+    check_standalone: pycv_here
     	$(MAKE) -C regtest testclean
     	$(MAKE) -C regtest checkfail
+    #just in case  pytest is still not installed we install it before the tests
+    check_python: pycv_here
+    	pip install pytest
+    	pytest
+    check: check_standalone check_python

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Pycv package ci #44

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Pycv package ci #44

Are you sure you want to change the base?

Uh oh!

Pycv package ci #44

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!