diff --git a/.github/workflows/build-and-publish-wheels.yml b/.github/workflows/build-and-publish-wheels.yml new file mode 100644 index 0000000..4b02a38 --- /dev/null +++ b/.github/workflows/build-and-publish-wheels.yml @@ -0,0 +1,131 @@ +name: Build and Publish Wheels + +on: + push: + branches: ['main'] + tags: + - "v*" # Build & publish on version tags (e.g. v1.2.3) + workflow_dispatch: # Allow manual trigger + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install Poetry + run: | + python -m pip install --upgrade pip + pip install poetry + + - name: Configure Poetry + run: poetry config virtualenvs.create true && poetry lock + + - name: Install dependencies + run: poetry install --no-interaction --no-root + + - name: Check C++ formatting + run: | + echo "Checking C++ code formatting..." + poetry run clang-format --dry-run --Werror $(find src/ -name '*.cpp' -o -name '*.hpp') + + - name: Build package wheel + run: | + pip install cibuildwheel + if [[ "$RUNNER_OS" == "Linux" ]]; then + cibuildwheel --output-dir dist --platform linux + elif [[ "$RUNNER_OS" == "macOS" ]]; then + CIBW_CONFIG_SETTINGS="setup-args=-Dmacos_arch=x86_64" cibuildwheel --output-dir dist --platform macos --archs x86_64 + CIBW_CONFIG_SETTINGS="setup-args=-Dmacos_arch=arm64" cibuildwheel --output-dir dist --platform macos --archs arm64 + fi + env: + MACOSX_DEPLOYMENT_TARGET: '15.0' + CIBW_SKIP: "pp* *-musllinux* cp36-* cp37-* cp38-*" + HOMEBREW_PREFIX: "" + HOMEBREW_CELLAR: "" + HOMEBREW_REPOSITORY: "" + PKG_CONFIG_PATH: "" + CIBW_ENVIRONMENT: "HOMEBREW_PREFIX='' HOMEBREW_CELLAR='' HOMEBREW_REPOSITORY='' PKG_CONFIG_PATH=''" + + - name: Upload built wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.os }} + path: dist/*.whl + + publish_testpypi: + name: Publish to TestPyPI + needs: build_wheels + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - uses: actions/download-artifact@v4 + with: + path: dist + - name: Upload to TestPyPI + run: | + pip install --upgrade pip poetry + ls -lrth dist + mkdir -p dist_flat + cp dist/**/*.whl dist_flat/ + rm -rf dist + mv dist_flat dist + ls -lrth dist + poetry config repositories.testpypi https://test.pypi.org/legacy/ + poetry publish -r testpypi -u __token__ -p ${{ secrets.TESTPYPI_API_TOKEN }} + + publish: + name: Publish to PyPI + needs: publish_testpypi + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Download distributions + uses: actions/download-artifact@v4 + with: + path: dist + + - uses: actions/download-artifact@v4 + with: + path: dist + - name: Upload to PyPI + run: | + pip install --upgrade pip poetry + mkdir -p dist_flat + cp dist/**/*.whl dist_flat/ + rm -rf dist + mv dist_flat dist + ls -lrth dist + poetry publish -u __token__ -p ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 0000000..7df040d --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,68 @@ +name: Build and Install (Python Matrix) + +on: + push: + branches: [ main ] + pull_request: + +jobs: + test: + name: python-${{ matrix.python }}-${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + python: ["3.9","3.10","3.11","3.12","3.13","3.14"] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Install Poetry + run: | + python -m pip install --upgrade pip + pip install poetry + + - name: Configure Poetry + run: poetry config virtualenvs.create true && poetry lock + + - name: Install dependencies + run: poetry install --no-interaction --no-root + + - name: Check C++ formatting + run: | + echo "Checking C++ code formatting..." + poetry run clang-format --dry-run --Werror $(find src/ -name '*.cpp' -o -name '*.hpp') + + - name: Build package wheel + run: | + poetry build + + - name: Patch RPATHs in wheel + run: | + if [[ "$RUNNER_OS" == "Linux" ]]; then + pip install auditwheel patchelf + auditwheel repair -w repaired_dist/ dist/*.whl + elif [[ "$RUNNER_OS" == "macOS" ]]; then + pip install delocate + delocate-listdeps dist/*.whl + delocate-wheel -w repaired_dist/ dist/*.whl + fi + + - name: Install package wheel + run: | + ls -lrth repaired_dist + poetry run pip install repaired_dist/*.whl + + - name: Run tests + run: | + poetry run pytest --maxfail=1 --disable-warnings -q diff --git a/.gitignore b/.gitignore index 7563f8b..63e687d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ *CMakeFiles *cmake_install.cmake *Makefile -*build +build # Pip files *.egg-info/ diff --git a/.gitmodules b/.gitmodules index a363dd4..fb208c4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "pybind11"] - path = pybind11 - url = https://github.com/pybind/pybind11.git [submodule "hipo"] path = hipo url = https://github.com/gavalian/hipo.git diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100644 index ca70ee9..0000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,41 +0,0 @@ -cmake_minimum_required(VERSION 3.15...4.0) - -#-----------------------------------------# -project(hipopybind) - -#-----------------------------------------# -# Add LZ4 dependency -add_compile_definitions(__LZ4__) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/hipo/lz4/lib) - -#-----------------------------------------# -# Add HIPO dependency -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/hipo) - -#-----------------------------------------# -# Add pybind11 dependency and module -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/pybind11) - -#-----------------------------------------# -file (GLOB SOURCE_FILES "hipo/hipo4/*.cpp") -file (GLOB HEADER_FILES "hipo/hipo4/*.h") -file (GLOB PYTHON_FILES "src/*.cpp" "src/*.h") -file (GLOB LZ4SRC_FILES "hipo/lz4/lib/*.c") -file (GLOB LZ4HDR_FILES "hipo/lz4/lib/*.h") - -pybind11_add_module(hipopybind - ${SOURCE_FILES} - ${HEADER_FILES} - ${PYTHON_FILES} - ${LZ4SRC_FILES} - ${LZ4HDR_FILES} -) - -# EXAMPLE_VERSION_INFO is defined by setup.py and passed into the C++ code as a -# define (VERSION_INFO) here. -target_compile_definitions(hipopybind - PRIVATE VERSION_INFO=${EXAMPLE_VERSION_INFO}) - -#-----------------------------------------# -# Link header files from subdirectories -target_include_directories(hipopybind PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index f444aae..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,6 +0,0 @@ -include README.md LICENSE pybind11/LICENSE -graft pybind11/include -graft pybind11/tools -graft hipo -graft src -global-include CMakeLists.txt *.cmake diff --git a/README.md b/README.md index 6c0431a..5d4d805 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,19 @@ # HipopyBind : HIPO PyBind11 Library +[![PyPI](https://img.shields.io/pypi/v/hipopybind.svg)](https://pypi.org/project/hipopybind/) +[![Python](https://github.com/mfmceneaney/hipopybind/actions/workflows/python.yml/badge.svg)](https://github.com/mfmceneaney/hipopybind/actions/workflows/python.yml) + +This project exposes in python the [hipo](https://github.com/gavalian/hipo) classes and a few +custom classes and functions from C++ via [pybind11](https://github.com/pybind/pybind11). ## Prerequisites -* Python >=3.7.3 -* A compiler with C++11 support -* Pip 10+ or CMake >= 3.15 (or 3.14+ on Windows, which was the first version to support VS 2019) +* macos or linux, windows is **not** supported +* Python >=3.9 +* A compiler with C++17 support * Ninja or Pip 10+ +* meson -## Installation +## :green_circle: Installation To install from PyPi run: @@ -15,28 +21,42 @@ To install from PyPi run: pip install hipopybind ``` -To compile the library from source run the following: +Compiling the library from source is **platform dependent**. +On macos run: ```bash git clone --recurse-submodules https://github.com/mfmceneaney/hipopybind.git cd hipopybind -cmake . -make +pip install poetry delocate +poetry build +delocate-wheel -w repaired_dist/ dist/*.whl +pip install repaired_dist/*.whl ``` -And add the following to your startup script: +On linux run: ```bash -export PYTHONPATH=$PYTHONPATH\:/path/to/hipopybind +git clone --recurse-submodules https://github.com/mfmceneaney/hipopybind.git +cd hipopybind +pip install poetry auditwheel +poetry build +auditwheel repair -w repaired_dist/ dist/*.whl +pip install repaired_dist/*.whl ``` -# Developer Note -For updating submodules run the following: +# :rocket: Getting Started +Run the tutorials via: ```bash -git submodule update --init --recursive +python3 tutorials/write.py +python3 tutorials/read.py ``` +# :red_circle: Troubleshooting +If you run into import errors because of undefined symbols when using the project along +with cppyy python libraries, try importing this library first since symbol visibility set to +hidden during compilation. + # Contact: matthew.mceneaney@duke.edu diff --git a/cross_x86_64_macos.txt b/cross_x86_64_macos.txt new file mode 100644 index 0000000..910be61 --- /dev/null +++ b/cross_x86_64_macos.txt @@ -0,0 +1,34 @@ +# cross_x86_64_macos.txt +# Cross compilation configuration for building x86_64 binaries on an arm64 macOS host. + +[binaries] +c = 'clang' +cpp = 'clang++' +ar = 'ar' +strip = 'strip' +pkgconfig = '/bin/true' # disables pkg-config lookups + +[properties] +# Tell Meson we’re cross-compiling for macOS x86_64 +needs_exe_wrapper = false +has_function_printf = true +skip_sanity_check = false +sys_root = '/' + +[built-in options] +c_args = ['-arch', 'x86_64'] +cpp_args = ['-arch', 'x86_64'] +c_link_args = ['-arch', 'x86_64'] +cpp_link_args = ['-arch', 'x86_64'] + +[host_machine] +system = 'darwin' +cpu_family = 'x86_64' +cpu = 'x86_64' +endian = 'little' + +[build_machine] +system = 'darwin' +cpu_family = 'arm64' +cpu = 'arm64' +endian = 'little' diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..066bcc4 --- /dev/null +++ b/meson.build @@ -0,0 +1,136 @@ +project( + 'hipopybind', + 'cpp', + version: '2.0.1', + license: 'MIT', + meson_version: '>=1.2', + default_options: [ + 'cpp_std=c++17', + 'buildtype=release', + 'python.install_env=auto', + ], +) + +# meson modules +pkg = import('pkgconfig') +fs = import('fs') + +#---------- Find python ----------# +python = import('python').find_installation(pure: false) + +# ---- Find Python and pybind11 ---- +pybind11_dep = dependency('pybind11', required: true) + +# ---- Try to find libhipo via pkg-config ---- +libhipo_dep = dependency('libhipo4', method: 'pkg-config', required: false) + +# ---- Set python install directory ---- +hb_pkg_dir = python.get_install_dir() / meson.project_name() + +# Set loader path +host_os = host_machine.system() +if host_os == 'darwin' + pybind_rpath = '@loader_path/lib' +else + # Linux and others + pybind_rpath = '$ORIGIN' +endif + +if not libhipo_dep.found() + message('libhipo not found via pkg-config. Attempting to build from source...') + + libhipo_src_dir = meson.project_source_root() / 'hipo' + libhipo_build_dir = meson.current_build_dir() / 'libhipo_build' + libhipo_install_dir = meson.project_source_root() + + # Run meson setup with workaround for macos x86_64 architecture if you are building on arm64 + macos_arch = get_option('macos_arch') + message('macos_arch = @0@'.format(macos_arch)) + if host_os == 'darwin' and macos_arch == 'x86_64' + message('Building for macos_arch==x86_64') + run_command('meson', 'setup', libhipo_build_dir, libhipo_src_dir, + '--prefix=' + libhipo_install_dir, + '--cross-file', 'cross_x86_64_macos.txt', + env: environment(), + check: true) + else + run_command('meson', 'setup', libhipo_build_dir, libhipo_src_dir, + '--prefix=' + libhipo_install_dir, + env: environment(), + check: true) + endif + + # Compile and install + run_command('meson', 'compile', '-C', libhipo_build_dir, check: true) + run_command('meson', 'install', '-C', libhipo_build_dir, check: true) + + # Assume libhipo_install_dir is a Meson File object or string path + hipo_include_dir = 'include' + hipo_lib_dir = libhipo_install_dir / 'lib' + hipo_lib_name = 'hipo4' + install_subdir( + hipo_include_dir, + install_dir: hb_pkg_dir + ) + install_subdir( + hipo_lib_dir, + install_dir: hb_pkg_dir + ) + if host_os != 'darwin' + install_data( + hipo_lib_dir / 'libhipo4.so', + install_dir: hb_pkg_dir + ) + run_command('ls', '-lrth', hipo_lib_dir, check: true) + lz4_path = hipo_lib_dir / 'liblz4.so.1.9.4' + if fs.is_file(lz4_path) + install_data( + lz4_path, + install_dir: hb_pkg_dir, + rename: 'liblz4.so' + ) + endif + endif + libhipo_dep = declare_dependency( + include_directories: include_directories(hipo_include_dir), + link_args: ['-L' + hipo_lib_dir, '-l' + hipo_lib_name] + ) +endif + +# ---- Build the Python extension ---- + +# Install the python init script so your python installation can find the package +install_data('src/__init__.py', install_dir: hb_pkg_dir) + +# Set link args based on os +py_mod_link_args = [] +if host_os == 'darwin' + py_mod_link_args += ['-Wl,-exported_symbol,_PyInit__core'] # explicitly export python module init function +else + py_mod_link_args += ['-Wl,--exclude-libs,ALL'] # don't re-export linked libs +endif + +# Compile the shared module +pybind_module = shared_module('_core', + 'src/hipopybind_module.cpp', + dependencies: [pybind11_dep, libhipo_dep], + cpp_args: [ + '-DVERSION_INFO="@0@"'.format(meson.project_version()), + '-fvisibility=hidden' + ], # hide all internal symbols + link_args: py_mod_link_args, + install: true, + install_dir: hb_pkg_dir, + install_rpath: pybind_rpath, + build_rpath: pybind_rpath, + name_prefix: '', + name_suffix: 'so', +) + +# Export a dependency object for use as a subproject +hipopybind_dep = declare_dependency( + link_with: pybind_module, +) + +# Export the dependency so superprojects can access it +meson.override_dependency('hipopybind_dep', hipopybind_dep) diff --git a/meson_options.txt b/meson_options.txt new file mode 100644 index 0000000..6486701 --- /dev/null +++ b/meson_options.txt @@ -0,0 +1,7 @@ +option('build_type', type: 'combo', choices: ['debug', 'release'], value: 'release') +option( + 'macos_arch', + type: 'string', + value: '', + description: 'Target macOS architecture (arm64 or x86_64)' +) \ No newline at end of file diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..0403456 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,256 @@ +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. + +[[package]] +name = "clang-format" +version = "17.0.6" +description = "Clang-Format is an LLVM-based code formatting tool" +optional = false +python-versions = "*" +groups = ["dev"] +files = [ + {file = "clang-format-17.0.6.tar.gz", hash = "sha256:50f082840d2e013160355ed63add4502884344371dda5af12ec0abe68cbc5a36"}, + {file = "clang_format-17.0.6-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:2c7364a50c4fdb8ce9acc4e0c21627e52f4eebee98ff2d8a19b6d4302d0be23b"}, + {file = "clang_format-17.0.6-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:195014a589fde9e2bec447ee1f2efd31f8c9f773b10aa66b510beae6997e6bc5"}, + {file = "clang_format-17.0.6-py2.py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cda40badfb01818ece739509d9cde678fc02660180cc1a55156782ef203704d"}, + {file = "clang_format-17.0.6-py2.py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:879e831c58a25a9b7527155032a6dc4758716ded69590911468a37629acb13d1"}, + {file = "clang_format-17.0.6-py2.py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:393f896db6155d6b8401ebae40df1f9a8cdf15d494d13fb775657c9ec609b586"}, + {file = "clang_format-17.0.6-py2.py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ccb6f5ce90f24ed0bb314d041a8edcc94d1279c1469669d5855be004d9d6caff"}, + {file = "clang_format-17.0.6-py2.py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a1323ca5322e0dead521223155fe2ae1ba81d50abab8e20aaac28f6a94f23b9"}, + {file = "clang_format-17.0.6-py2.py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:5476f8fba40e4330a4704681386d78751ced0ecbd050bd0687817cca01d4e167"}, + {file = "clang_format-17.0.6-py2.py3-none-musllinux_1_1_i686.whl", hash = "sha256:a0f744b056cb1595efdb7d2b83a7d73370e506e17fcaa68cd884c2ed029ae0fd"}, + {file = "clang_format-17.0.6-py2.py3-none-musllinux_1_1_ppc64le.whl", hash = "sha256:2ddc8b6237520d26d78489e3bb876243d87c3629eb3cd40e1df0c8c6e355d949"}, + {file = "clang_format-17.0.6-py2.py3-none-musllinux_1_1_s390x.whl", hash = "sha256:afc29c4413b5f2f885347f4bdbb7fe81f595faeceafa640c9e67a2d9aa2c7134"}, + {file = "clang_format-17.0.6-py2.py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:33c4f1975a6a0a76e5b85165c510c46ae1155f82477a5730e29799e43d78c83a"}, + {file = "clang_format-17.0.6-py2.py3-none-win32.whl", hash = "sha256:edd55b840fa6edcdafb1651c3c24c6ea8d911e73be30373e7e8e5741cb585464"}, + {file = "clang_format-17.0.6-py2.py3-none-win_amd64.whl", hash = "sha256:9407f0f4cb5a26b96af38bb2261f1c4015127f4d87ce46a61bb3a3c2a3d4f3cc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["dev"] +markers = "sys_platform == \"win32\"" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +groups = ["dev"] +markers = "python_version < \"3.11\"" +files = [ + {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, + {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "meson" +version = "1.9.1" +description = "A high performance build system" +optional = false +python-versions = ">=3.7" +groups = ["main", "dev"] +files = [ + {file = "meson-1.9.1-py3-none-any.whl", hash = "sha256:f824ab770c041a202f532f69e114c971918ed2daff7ea56583d80642564598d0"}, +] + +[package.extras] +ninja = ["ninja (>=1.8.2)"] +progress = ["tqdm"] +typing = ["mypy", "typing_extensions ; python_version < \"3.8\""] + +[[package]] +name = "meson-python" +version = "0.18.0" +description = "Meson Python build backend (PEP 517)" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "meson_python-0.18.0-py3-none-any.whl", hash = "sha256:3b0fe051551cc238f5febb873247c0949cd60ded556efa130aa57021804868e2"}, + {file = "meson_python-0.18.0.tar.gz", hash = "sha256:c56a99ec9df669a40662fe46960321af6e4b14106c14db228709c1628e23848d"}, +] + +[package.dependencies] +meson = [ + {version = ">=0.64.0", markers = "python_version < \"3.12\""}, + {version = ">=1.2.3", markers = "python_version >= \"3.12\""}, +] +packaging = ">=23.2" +pyproject-metadata = ">=0.9.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +docs = ["furo (>=2024.08.06)", "sphinx (>=8.1.0,<8.2.0)", "sphinx-copybutton (>=0.5.0)", "sphinx-design (>=0.1.0)", "sphinxext-opengraph (>=0.7.0)"] +test = ["build", "cython (>=3.0.3)", "pytest (>=6.0)", "pytest-cov[toml]", "pytest-mock", "typing-extensions (>=3.7.4) ; python_version < \"3.11\"", "wheel"] + +[[package]] +name = "ninja" +version = "1.13.0" +description = "Ninja is a small build system with a focus on speed" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "ninja-1.13.0-py3-none-macosx_10_9_universal2.whl", hash = "sha256:fa2a8bfc62e31b08f83127d1613d10821775a0eb334197154c4d6067b7068ff1"}, + {file = "ninja-1.13.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3d00c692fb717fd511abeb44b8c5d00340c36938c12d6538ba989fe764e79630"}, + {file = "ninja-1.13.0-py3-none-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:be7f478ff9f96a128b599a964fc60a6a87b9fa332ee1bd44fa243ac88d50291c"}, + {file = "ninja-1.13.0-py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:60056592cf495e9a6a4bea3cd178903056ecb0943e4de45a2ea825edb6dc8d3e"}, + {file = "ninja-1.13.0-py3-none-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:1c97223cdda0417f414bf864cfb73b72d8777e57ebb279c5f6de368de0062988"}, + {file = "ninja-1.13.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fb46acf6b93b8dd0322adc3a4945452a4e774b75b91293bafcc7b7f8e6517dfa"}, + {file = "ninja-1.13.0-py3-none-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4be9c1b082d244b1ad7ef41eb8ab088aae8c109a9f3f0b3e56a252d3e00f42c1"}, + {file = "ninja-1.13.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:6739d3352073341ad284246f81339a384eec091d9851a886dfa5b00a6d48b3e2"}, + {file = "ninja-1.13.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:11be2d22027bde06f14c343f01d31446747dbb51e72d00decca2eb99be911e2f"}, + {file = "ninja-1.13.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:aa45b4037b313c2f698bc13306239b8b93b4680eb47e287773156ac9e9304714"}, + {file = "ninja-1.13.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5f8e1e8a1a30835eeb51db05cf5a67151ad37542f5a4af2a438e9490915e5b72"}, + {file = "ninja-1.13.0-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:3d7d7779d12cb20c6d054c61b702139fd23a7a964ec8f2c823f1ab1b084150db"}, + {file = "ninja-1.13.0-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:d741a5e6754e0bda767e3274a0f0deeef4807f1fec6c0d7921a0244018926ae5"}, + {file = "ninja-1.13.0-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:e8bad11f8a00b64137e9b315b137d8bb6cbf3086fbdc43bf1f90fd33324d2e96"}, + {file = "ninja-1.13.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b4f2a072db3c0f944c32793e91532d8948d20d9ab83da9c0c7c15b5768072200"}, + {file = "ninja-1.13.0-py3-none-win32.whl", hash = "sha256:8cfbb80b4a53456ae8a39f90ae3d7a2129f45ea164f43fadfa15dc38c4aef1c9"}, + {file = "ninja-1.13.0-py3-none-win_amd64.whl", hash = "sha256:fb8ee8719f8af47fed145cced4a85f0755dd55d45b2bddaf7431fa89803c5f3e"}, + {file = "ninja-1.13.0-py3-none-win_arm64.whl", hash = "sha256:3c0b40b1f0bba764644385319028650087b4c1b18cdfa6f45cb39a3669b81aa9"}, + {file = "ninja-1.13.0.tar.gz", hash = "sha256:4a40ce995ded54d9dc24f8ea37ff3bf62ad192b547f6c7126e7e25045e76f978"}, +] + +[[package]] +name = "packaging" +version = "25.0" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, + {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, +] + +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pybind11" +version = "3.0.1" +description = "Seamless operability between C++11 and Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "pybind11-3.0.1-py3-none-any.whl", hash = "sha256:aa8f0aa6e0a94d3b64adfc38f560f33f15e589be2175e103c0a33c6bce55ee89"}, + {file = "pybind11-3.0.1.tar.gz", hash = "sha256:9c0f40056a016da59bab516efb523089139fcc6f2ba7e4930854c61efb932051"}, +] + +[package.extras] +global = ["pybind11-global (==3.0.1)"] + +[[package]] +name = "pyproject-metadata" +version = "0.9.1" +description = "PEP 621 metadata parsing" +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "pyproject_metadata-0.9.1-py3-none-any.whl", hash = "sha256:ee5efde548c3ed9b75a354fc319d5afd25e9585fa918a34f62f904cc731973ad"}, + {file = "pyproject_metadata-0.9.1.tar.gz", hash = "sha256:b8b2253dd1b7062b78cf949a115f02ba7fa4114aabe63fa10528e9e1a954a816"}, +] + +[package.dependencies] +packaging = ">=19.0" + +[package.extras] +docs = ["furo (>=2023.9.10)", "myst-parser", "sphinx (>=7.0)", "sphinx-autodoc-typehints", "sphinx-autodoc-typehints (>=1.10.0)"] +test = ["exceptiongroup ; python_version < \"3.11\"", "pytest (>=6.2.4)", "pytest-cov[toml] (>=2)", "tomli (>=1.0.0) ; python_version < \"3.11\""] + +[[package]] +name = "pytest" +version = "8.3.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820"}, + {file = "pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.5,<2" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +groups = ["dev"] +markers = "python_version < \"3.11\"" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.7.1" +description = "Backported and Experimental Type Hints for Python 3.7+" +optional = false +python-versions = ">=3.7" +groups = ["dev"] +markers = "python_version < \"3.11\"" +files = [ + {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, + {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, +] + +[metadata] +lock-version = "2.1" +python-versions = ">=3.9" +content-hash = "cf440429703e5be8b53694a06daab6878ffa84725ee243610cf39bc44844f867" diff --git a/pybind11 b/pybind11 deleted file mode 160000 index 55b1357..0000000 --- a/pybind11 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 55b1357d8ed93faf65ef3925ee38dddf79d3842f diff --git a/pyproject.toml b/pyproject.toml index ae13dad..25ec6bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,25 +1,29 @@ [build-system] requires = [ - "setuptools>61.0", - "wheel", - "ninja", - "cmake>=3.15", + "meson>=1.2", + "meson-python", + "pybind11", + "ninja>=1.8.2" ] -build-backend = "setuptools.build_meta" +build-backend = 'mesonpy' -[tool.isort] -profile = "black" +[tool.poetry] +name = "hipopybind" +version = "2.0.1" +description = "Python bindings for hipo using pybind11" +authors = ["Matthew McEneaney "] +repository = "https://github.com/mfmceneaney/hipopybind.git" +readme = "README.md" +license = "MIT" +classifiers = ["Programming Language :: Python :: 3"] -[tool.pytest.ini_options] -minversion = "6.0" -addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"] -xfail_strict = true -filterwarnings = ["error"] -testpaths = ["tests"] +[tool.poetry.dependencies] +python = ">=3.9" +meson = ">=1.2" +ninja = ">=1.8.2" +pybind11 = ">=3.0.1" -[tool.cibuildwheel] -test-command = "pytest {project}/tests" -test-extras = ["test"] -test-skip = ["*universal2:arm64"] -# Setuptools bug causes collision between pypy and cpython artifacts -before-build = "rm -rf {project}/build" +[tool.poetry.group.dev.dependencies] +pytest = "^8.0.0" +clang-format = "^17.0.0" # For local C++ formatting +meson-python = "^0.18.0" diff --git a/setup.py b/setup.py deleted file mode 100644 index 945edd6..0000000 --- a/setup.py +++ /dev/null @@ -1,148 +0,0 @@ -import os -import re -import subprocess -import sys -from pathlib import Path - -from setuptools import Extension, setup -from setuptools.command.build_ext import build_ext - -#NOTE: Build file based off the one in https://github.com/pybind/cmake_example.git - -# Convert distutils Windows platform specifiers to CMake -A arguments -PLAT_TO_CMAKE = { - "win32": "Win32", - "win-amd64": "x64", - "win-arm32": "ARM", - "win-arm64": "ARM64", -} - - -# A CMakeExtension needs a sourcedir instead of a file list. -# The name must be the _single_ output extension from the CMake build. -# If you need multiple extensions, see scikit-build. -class CMakeExtension(Extension): - def __init__(self, name: str, sourcedir: str = "") -> None: - super().__init__(name, sources=[]) - self.sourcedir = os.fspath(Path(sourcedir).resolve()) - - -class CMakeBuild(build_ext): - def build_extension(self, ext: CMakeExtension) -> None: - # Must be in this form due to bug in .resolve() only fixed in Python 3.10+ - ext_fullpath = Path.cwd() / self.get_ext_fullpath(ext.name) - extdir = ext_fullpath.parent.resolve() - - # Using this requires trailing slash for auto-detection & inclusion of - # auxiliary "native" libs - - debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug - cfg = "Debug" if debug else "Release" - - # CMake lets you override the generator - we need to check this. - # Can be set with Conda-Build, for example. - cmake_generator = os.environ.get("CMAKE_GENERATOR", "") - - # Set Python_EXECUTABLE instead if you use PYBIND11_FINDPYTHON - # EXAMPLE_VERSION_INFO shows you how to pass a value into the C++ code - # from Python. - cmake_args = [ - f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}{os.sep}", - f"-DPYTHON_EXECUTABLE={sys.executable}", - f"-DCMAKE_BUILD_TYPE={cfg}", # not used on MSVC, but no harm - ] - build_args = [] - # Adding CMake arguments set as environment variable - # (needed e.g. to build for ARM OSx on conda-forge) - if "CMAKE_ARGS" in os.environ: - cmake_args += [item for item in os.environ["CMAKE_ARGS"].split(" ") if item] - - # In this example, we pass in the version to C++. You might not need to. - cmake_args += [f"-DEXAMPLE_VERSION_INFO={self.distribution.get_version()}"] - - if self.compiler.compiler_type != "msvc": - # Using Ninja-build since it a) is available as a wheel and b) - # multithreads automatically. MSVC would require all variables be - # exported for Ninja to pick it up, which is a little tricky to do. - # Users can override the generator with CMAKE_GENERATOR in CMake - # 3.15+. - if (not cmake_generator or cmake_generator == "Ninja") and not sys.platform.startswith('linux'): #NOTE: #DEBUGGING: ADDED FOR BUILD TO WORK ON IFARM - try: - import ninja - - ninja_executable_path = Path(ninja.BIN_DIR) / "ninja" - cmake_args += [ - "-GNinja", - f"-DCMAKE_MAKE_PROGRAM:FILEPATH={ninja_executable_path}", - ] - except ImportError: - pass - - else: - # Single config generators are handled "normally" - single_config = any(x in cmake_generator for x in {"NMake", "Ninja"}) - - # CMake allows an arch-in-generator style for backward compatibility - contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"}) - - # Specify the arch if using MSVC generator, but only if it doesn't - # contain a backward-compatibility arch spec already in the - # generator name. - if not single_config and not contains_arch: - cmake_args += ["-A", PLAT_TO_CMAKE[self.plat_name]] - - # Multi-config generators have a different way to specify configs - if not single_config: - cmake_args += [ - f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}" - ] - build_args += ["--config", cfg] - - if sys.platform.startswith("darwin"): - # Cross-compile support for macOS - respect ARCHFLAGS if set - archs = re.findall(r"-arch (\S+)", os.environ.get("ARCHFLAGS", "")) - if archs: - cmake_args += ["-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))] - - # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level - # across all generators. - if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ: - # self.parallel is a Python 3 only way to set parallel jobs by hand - # using -j in the build_ext call, not supported by pip or PyPA-build. - if hasattr(self, "parallel") and self.parallel: - # CMake 3.12+ only. - build_args += [f"-j{self.parallel}"] - - build_temp = Path(self.build_temp) / ext.name - if not build_temp.exists(): - build_temp.mkdir(parents=True) - - subprocess.run( - ["cmake", ext.sourcedir, *cmake_args], cwd=build_temp, check=True - ) - subprocess.run( - ["cmake", "--build", ".", *build_args], cwd=build_temp, check=True - ) - -#NOTE: Define function for reading README file into long description -def read(fname): - return open(os.path.join(os.path.dirname(__file__), fname)).read() - -# The information here can also be placed in setup.cfg - better separation of -# logic and declaration, and simpler if you include description/version in a file. -setup( - name="hipopybind", - version="1.1.3", - author="Matthew McEneaney", - author_email="matthew.mceneaney@duke.edu", - license="MIT", - url="https://github.com/mfmceneaney/hipopybind.git", - description="A HIPO python library using PyBind11 and CMake", - long_description=read('README.md'), - long_description_content_type='text/markdown', - ext_modules=[CMakeExtension("hipopybind")], - cmdclass={"build_ext": CMakeBuild}, - zip_safe=False, - extras_require={"test": ["pytest>=6.0"]}, - python_requires=">=3.7", -) diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..7daa176 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,2 @@ +__version__="2.0.1" +from ._core import * diff --git a/src/hipopybind_module.cpp b/src/hipopybind_module.cpp new file mode 100644 index 0000000..55e5c48 --- /dev/null +++ b/src/hipopybind_module.cpp @@ -0,0 +1,1446 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define STRINGIFY(x) #x +#define MACRO_STRINGIFY(x) STRINGIFY(x) + +// NOTE: Shared object will not load if you simultaneously +// define classes to bind within this file and bind hipo +// classes or classes from another cpp project. + +//--------------------------------------------------// +// Get array functions - Matthew McEneaney + +std::vector _getInts(hipo::bank bank, const char *name) noexcept { + hipo::schema &bankSchema = bank.getSchema(); + int item = bankSchema.getEntryOrder(name); + std::vector arr(0); + if (bankSchema.getEntryType(item) == 3) { + for (int index = 0; index < bank.getRows(); index++) { + int offset = bankSchema.getOffset(item, index, bank.getRows()); + arr.push_back((int)bank.getIntAt(offset)); + } + } + return arr; +} + +std::vector _getShorts(hipo::bank bank, const char *name) noexcept { + hipo::schema &bankSchema = bank.getSchema(); + int item = bankSchema.getEntryOrder(name); + std::vector arr(0); + if (bankSchema.getEntryType(item) == 2) { + for (int index = 0; index < bank.getRows(); index++) { + int offset = bankSchema.getOffset(item, index, bank.getRows()); + arr.push_back((int)bank.getShortAt(offset)); + } + } + return arr; +} + +std::vector _getBytes(hipo::bank bank, const char *name) noexcept { + hipo::schema &bankSchema = bank.getSchema(); + int item = bankSchema.getEntryOrder(name); + std::vector arr(0); + if (bankSchema.getEntryType(item) == 1) { + for (int index = 0; index < bank.getRows(); index++) { + int offset = bankSchema.getOffset(item, index, bank.getRows()); + arr.push_back((int)bank.getByteAt(offset)); + } + } + return arr; +} + +std::vector _getFloats(hipo::bank bank, const char *name) noexcept { + hipo::schema &bankSchema = bank.getSchema(); + int item = bankSchema.getEntryOrder(name); + std::vector arr(0); + if (bankSchema.getEntryType(item) == 4) { + for (int index = 0; index < bank.getRows(); index++) { + int offset = bankSchema.getOffset(item, index, bank.getRows()); + arr.push_back(bank.getFloatAt(offset)); + } + } + return arr; +} + +std::vector _getDoubles(hipo::bank bank, const char *name) noexcept { + hipo::schema &bankSchema = bank.getSchema(); + int item = bankSchema.getEntryOrder(name); + std::vector arr(0); + if (bankSchema.getEntryType(item) == 5) { + for (int index = 0; index < bank.getRows(); index++) { + int offset = bankSchema.getOffset(item, index, bank.getRows()); + arr.push_back(bank.getDoubleAt(offset)); + } + } + return arr; +} + +std::vector _getLongs(hipo::bank bank, const char *name) noexcept { + hipo::schema &bankSchema = bank.getSchema(); + int item = bankSchema.getEntryOrder(name); + std::vector arr(0); + if (bankSchema.getEntryType(item) == 8) { + for (int index = 0; index < bank.getRows(); index++) { + int offset = bankSchema.getOffset(item, index, bank.getRows()); + arr.push_back(bank.getLongAt(offset)); + } + } + return arr; +} + +//--------------------------------------------------// + +void _putInts(hipo::bank &bank, const char *name, std::vector arr) { + hipo::schema &bankSchema = bank.getSchema(); + int item = bankSchema.getEntryOrder(name); + for (std::size_t index = 0; index < arr.size(); index++) { + int offset = bankSchema.getOffset(item, index, bank.getRows()); + bank.putIntAt(offset, arr[index]); + } +} +void _putShorts(hipo::bank &bank, const char *name, std::vector arr) { + hipo::schema &bankSchema = bank.getSchema(); + int item = bankSchema.getEntryOrder(name); + for (std::size_t index = 0; index < arr.size(); index++) { + int offset = bankSchema.getOffset(item, index, bank.getRows()); + bank.putShortAt(offset, arr[index]); + } +} +void _putBytes(hipo::bank &bank, const char *name, std::vector arr) { + hipo::schema &bankSchema = bank.getSchema(); + int item = bankSchema.getEntryOrder(name); + for (std::size_t index = 0; index < arr.size(); index++) { + int offset = bankSchema.getOffset(item, index, bank.getRows()); + bank.putByteAt(offset, arr[index]); + } +} +void _putFloats(hipo::bank &bank, const char *name, std::vector arr) { + hipo::schema &bankSchema = bank.getSchema(); + int item = bankSchema.getEntryOrder(name); + for (std::size_t index = 0; index < arr.size(); index++) { + int offset = bankSchema.getOffset(item, index, bank.getRows()); + bank.putFloatAt(offset, arr[index]); + } +} +void _putDoubles(hipo::bank &bank, const char *name, std::vector arr) { + hipo::schema &bankSchema = bank.getSchema(); + int item = bankSchema.getEntryOrder(name); + for (std::size_t index = 0; index < arr.size(); index++) { + int offset = bankSchema.getOffset(item, index, bank.getRows()); + bank.putDoubleAt(offset, arr[index]); + } +} +void _putLongs(hipo::bank &bank, const char *name, std::vector arr) { + hipo::schema &bankSchema = bank.getSchema(); + int item = bankSchema.getEntryOrder(name); + for (std::size_t index = 0; index < arr.size(); index++) { + int offset = bankSchema.getOffset(item, index, bank.getRows()); + bank.putLongAt(offset, arr[index]); + } +} + +//--------------------------------------------------// + +//--------------------------------------------------// +// Custom classes + +class HipoFileIterator { + +private: + int batchsize; + int nbanks; + int index; + std::vector filenames; + std::vector banknames; + std::vector banklist; + std::vector> typelist; + std::vector> itemlist; + hipo::reader reader; + // hipo::writer writer; + hipo::dictionary dict; + hipo::event event; + + std::vector>> vec_double; + std::vector>> vec_float; + std::vector>> vec_int; + std::vector>> vec_long; + std::vector>> vec_short; + std::vector>> vec_byte; + + std::map item_index_map; + std::map item_type_map; + std::string separator; + std::vector item_indices; + std::vector tags; + +protected: + void protected_method(); + +public: + HipoFileIterator(std::vector &__filenames__, + std::vector &__banknames__, int __batchsize__, + std::vector &__tags__) { + filenames = __filenames__; + banknames = __banknames__; + nbanks = banknames.size(); + batchsize = __batchsize__; + separator = "_"; + item_indices = + std::vector(9); // NOTE: Only 6 types, but they only go up to 8 and + // it's easier to use the type int as the index. + tags = __tags__; + + // Check length of filenames + if (filenames.size() == 0) + throw std::invalid_argument( + "HipoFileIterator: length of filenames must be > 0"); + + // Check batchsize + if (batchsize <= 0) + throw std::invalid_argument("HipoFileIterator: batch_size must be > 0"); + + index = -1; + + // Set tags before opening files + int tags_size = tags.size(); + for (int i = 0; i < tags_size; i++) { + reader.setTags(tags.at(i)); + } + + // Open first file and set bank names to all banks if none specified + open(); + if (nbanks == 0) { + banknames = dict.getSchemaList(); + nbanks = banknames.size(); + } + + // Loop bank names and create banks and get entry names and types + typelist = std::vector>(0); + itemlist = std::vector>(0); + for (int i = 0; i < nbanks; i++) { + const char *bankname = banknames.at(i).c_str(); + hipo::schema &schema = dict.getSchema(bankname); + hipo::bank bank = hipo::bank(schema); + banklist.push_back(bank); + event.getStructure(bank); + + // Loop bank entries and get names and types + int nentries = schema.getEntries(); + std::vector vec_entrytype = std::vector(0); + std::vector vec_entryname = std::vector( + 0); // NOTE: IMPORTANT THAT THIS IS ZERO. IF YOU SET TO NENTRIES YOU + // MESS UP PUSHBACK AND GET A VECTOR TWICE AS LONG AS YOU WANT. + for (int j = 0; j < nentries; j++) { + int entrytype = schema.getEntryType(j); + std::string entryname = schema.getEntryName(j); + vec_entrytype.push_back(entrytype); + vec_entryname.push_back(entryname); + + // Update bank+item to index map + item_index_map.insert(std::make_pair(bankname + separator + entryname, + item_indices[entrytype])); + item_type_map.insert( + std::make_pair(bankname + separator + entryname, entrytype)); + item_indices.at(entrytype) = item_indices.at(entrytype) + 1; + } + typelist.push_back(vec_entrytype); + itemlist.push_back(vec_entryname); + } + reset(); + + } // HipoFileIterator() //NOTE: //TODO: Could also define a init method where + // you specify bank entries and then parse in python... and could also + // figure out how to include cuts.... + + virtual ~HipoFileIterator() = default; + + /** + * Open the next file in this.filenames if the end of this.filenames has not + * yet been reached. + * @return bool + */ + bool open() { + index++; + int filenames_size = filenames.size(); + if (index >= filenames_size) + return false; + const char *filename = filenames.at(index).c_str(); + reader.open(filename); + reader.readDictionary(dict); // TODO: Figure out how to check file here.... + return true; + } + + /** + * Resets the vectors used for storing batch data of all available types. + * @exception std::out_of_range + */ + void reset() { + try { + vec_double = + std::vector>>(item_indices.at(5)); + vec_float = + std::vector>>(item_indices.at(4)); + vec_int = std::vector>>(item_indices.at(3)); + vec_long = + std::vector>>(item_indices.at(8)); + vec_short = + std::vector>>(item_indices.at(2)); + vec_byte = std::vector>>(item_indices.at(1)); + } catch (const std::out_of_range &e) { + std::cout << e.what() << std::endl; // DEBUGGING + } + return; + } + + /** + * Move reader to given event number in file. + * @param int n + * @return bool + */ + bool gotoEvent(int n) { // TODO: CHECK THIS METHOD AND DO A REWIND METHOD TOO. + return reader.gotoEvent(n); + } + + /** + * Add a reader tag. + * @param int tag + */ + void setTags(int tag) { reader.setTags(tag); } + + /** + * Get reader tags. + * @return std::vector tags + */ + std::vector getTags() { return tags; } + + /** + * Recursive function which loops file(s) to read data into next batch. + * @param int __counter__ + * @return bool + */ + bool next(int __counter__ = 0) { + + int counter = __counter__; + + // Loop events + while (reader.next()) { + reader.read(event); + + // Loop banks + for (int i = 0; i < nbanks; i++) { + + // Read bank + std::string bankname = banknames.at(i); + hipo::bank bank = banklist.at(i); + event.getStructure(bank); + + // Get entry names and types + std::vector entrytypes = typelist.at(i); + std::vector entrynames = + itemlist.at(i); // TODO: COULD ALSO COMPARE TIME WITH SCHEMALIST... + int nentries = entrytypes.size(); + + // Loop entries + for (int j = 0; j < nentries; j++) { + int type = entrytypes.at(j); + const char *name = entrynames.at(j).c_str(); + int entryindex = item_index_map.at(bankname + separator + name); + switch (type) { + case 1: + vec_byte.at(entryindex).push_back(_getBytes(bank, name)); + break; // NOTE COULD INSERT INTO PREEXISTING VECTOR? AND THEN + // INITIALIZE VECTOR WITH CORRECT DIMENSIONS ON FIRST TWO + // AXES... + case 2: + vec_short.at(entryindex).push_back(_getShorts(bank, name)); + break; // TODO: ALSO NEED TO DEAL WITH CASE WHERE SOME EVENTS DON'T + // HAVE ALL THE BANKS. + case 3: + vec_int.at(entryindex).push_back(_getInts(bank, name)); + break; // TODO: Make sure this will modify in place. + case 4: + vec_float.at(entryindex).push_back(_getFloats(bank, name)); + break; // TODO: Figure out how to add empty events for banks that + // don't occur in a given event... -> GetDoubles etc should + // return empty array if bank is empty in event and schema is + // still there so this should be fine, in principle. + case 5: + vec_double.at(entryindex).push_back(_getDoubles(bank, name)); + break; + case 8: + vec_long.at(entryindex).push_back(_getLongs(bank, name)); + break; + default: + throw pybind11::type_error( + "HipoFileIterator: Invalid type " + std::to_string(type) + + " for entry: " + entrynames.at(j) + "\n"); + } + } // for (int j = 0; j= batchsize) { + int filenames_size = filenames.size(); + if (!reader.hasNext() && index >= filenames_size - 1) + return false; + return true; + } + } + // Move on to next file if batch is not yet complete + if (open()) { + return next( + counter); // NOTE: Recursive function //NOTE: If you do not return + // here, the original function keeps going after the + // recursive call and returns false and so ends iteration. + } + // TODO: NOTE: THIS SHOULD NOT RAISE A STOP ITERATION BECAUSE YOU STILL NEED + // TO GET THE BATCH DATA IN PYTHON + return false; // NOTE: Only happens if you've reached the end of all files. + } + + /** + * Get type int for given bank name and entry name. + * @param std::string bankname + * @param std::string item + * @return int i + */ + int getType(std::string bankname, std::string item) { + return item_type_map.at( + bankname + separator + + item); // NOTE: //TODO: COULD ADD pybind11::attribute_error here and try + // block. + } + + /** + * Get batch array of doubles for given bank name and entry name. + * @param std::string bankname + * @param std::string item + * @return std::vector> + */ + std::vector> getDoubles(std::string bankname, + std::string item) const noexcept { + int i = item_index_map.at(bankname + separator + item); + return vec_double.at(i); + } + + /** + * Get batch array of floats for given bank name and entry name. + * @param std::string bankname + * @param std::string item + * @return std::vector> + */ + std::vector> getFloats(std::string bankname, + std::string item) const noexcept { + int i = item_index_map.at(bankname + separator + item); + return vec_float.at(i); + } + + /** + * Get batch array of ints for given bank name and entry name. + * @param std::string bankname + * @param std::string item + * @return std::vector> + */ + std::vector> getInts(std::string bankname, + std::string item) const noexcept { + int i = item_index_map.at(bankname + separator + item); + return vec_int.at(i); + } + + /** + * Get batch array of longs for given bank name and entry name. + * @param std::string bankname + * @param std::string item + * @return std::vector> + */ + std::vector> getLongs(std::string bankname, + std::string item) const noexcept { + int i = item_index_map.at(bankname + separator + item); + return vec_long.at(i); + } + + /** + * Get batch array of shorts for given bank name and entry name. + * @param std::string bankname + * @param std::string item + * @return std::vector> + */ + std::vector> getShorts(std::string bankname, + std::string item) const noexcept { + int i = item_index_map.at(bankname + separator + item); + return vec_short.at(i); + } + + /** + * Get batch array of bytes for given bank name and entry name. + * @param std::string bankname + * @param std::string item + * @return std::vector> + */ + std::vector> getBytes(std::string bankname, + std::string item) const noexcept { + int i = item_index_map.at(bankname + separator + item); + return vec_byte.at(i); + } + + /* + * Get methods for class attributes + */ + int getBatchsize() { return batchsize; } + int getNBanks() { return nbanks; } + int getIndex() { return index; } + std::vector getFilenames() { return filenames; } + std::vector getBanknames() { return banknames; } + std::vector getBanklist() { return banklist; } + std::vector> getTypelist() { return typelist; } + std::vector> getItemlist() { return itemlist; } + hipo::reader getReader() { return reader; } + // hipo::writer getWriter() { return writer; } + hipo::dictionary getDict() { return dict; } + hipo::event getEvent() { return event; } + +}; // class HipoFileIterator + +//--------------------------------------------------// +// HipoFileIterator Trampoline class +template +class PyHipoFileIterator : public HipoFileIteratorBase { +public: + using HipoFileIteratorBase::HipoFileIteratorBase; // Inherit constructors +}; + +//--------------------------------------------------// +// HIPO Trampoline classes + +// NOTE: utils, benchmark classes throw compilation error if you try to use +// trampoline class. + +template +class PyDatastream : public DatastreamBase { +public: + using DatastreamBase::DatastreamBase; // Inherit constructors + long size() override { PYBIND11_OVERRIDE_PURE(long, DatastreamBase, size, ); } + long position() override { + PYBIND11_OVERRIDE_PURE(long, DatastreamBase, position, ); + } + long position(long pos) override { + PYBIND11_OVERRIDE_PURE(long, DatastreamBase, position, pos); + } + void open(const char *filename) override { + PYBIND11_OVERRIDE_PURE(void, DatastreamBase, open, filename); + } + int read(char *s, int size) override { + PYBIND11_OVERRIDE_PURE(int, DatastreamBase, read, s, size); + } +}; + +// NOTE: datastreamLocalFile, datastreamXrootd classes thorw compilation error +// if you try to use trampoline class. + +template class PySchema : public SchemaBase { +public: + using SchemaBase::SchemaBase; // Inherit constructors +}; + +template +class PyDictionary : public DictionaryBase { +public: + using DictionaryBase::DictionaryBase; // Inherit constructors +}; + +template +class PyStructure : public StructureBase { +public: + using StructureBase::StructureBase; // Inherit constructors + void notify() override { + PYBIND11_OVERRIDE_PURE(void, StructureBase, notify, ); + } +}; + +template +class PyBank : public PyStructure { +public: + using PyStructure::PyStructure; // Inherit constructors +}; + +template class PyEvent : public EventBase { +public: + using EventBase::EventBase; // Inherit constructors +}; + +// NOTE: readerIndex, reader, data, record classes throw compilation error if +// you try to use trampoline class. + +template +class PyRecordBuilder : public RecordBuilderBase { +public: + using RecordBuilderBase::RecordBuilderBase; // Inherit constructors +}; + +template class PyWriter : public WriterBase { +public: + using WriterBase::WriterBase; // Inherit constructors +}; + +//--------------------------------------------------// +// Python Module Bindings + +namespace py = pybind11; +PYBIND11_MODULE(_core, m) { + + // Add functions to module + m.def("getInts", &_getInts); + m.def("getShorts", &_getShorts); + m.def("getBytes", &_getBytes); + m.def("getFloats", &_getFloats); + m.def("getDoubles", &_getDoubles); + m.def("getLongs", &_getLongs); + m.def("putInts", &_putInts); + m.def("putShorts", &_putShorts); + m.def("putBytes", &_putBytes); + m.def("putFloats", &_putFloats); + m.def("putDoubles", &_putDoubles); + m.def("putLongs", &_putLongs); + + // ADDED BEGIN + //----------------------------------------------------------------------// + // Bind HipoFileIterator + py::class_> hipofileiterator( + m, "HipoFileIterator"); + hipofileiterator.def(py::init([](std::vector &filenames, + std::vector &banknames, + int batchsize, std::vector &tags) { + return new HipoFileIterator(filenames, banknames, batchsize, tags); + })); + hipofileiterator.def("open", &HipoFileIterator::open); + hipofileiterator.def("reset", &HipoFileIterator::reset); + hipofileiterator.def("next", &HipoFileIterator::next); + hipofileiterator.def("gotoEvent", &HipoFileIterator::gotoEvent); + hipofileiterator.def( + "setTags", + &HipoFileIterator::setTags); // TODO: ADD OPTION IN INIT TO SET TAGS... + hipofileiterator.def("getType", &HipoFileIterator::getType); + hipofileiterator.def("getDoubles", &HipoFileIterator::getDoubles); + hipofileiterator.def("getFloats", &HipoFileIterator::getFloats); + hipofileiterator.def("getInts", &HipoFileIterator::getInts); + hipofileiterator.def("getLongs", &HipoFileIterator::getLongs); + hipofileiterator.def("getShorts", &HipoFileIterator::getShorts); + hipofileiterator.def("getBytes", &HipoFileIterator::getBytes); + + hipofileiterator.def_property_readonly("batchsize", + &HipoFileIterator::getBatchsize); + hipofileiterator.def_property_readonly("nbanks", + &HipoFileIterator::getNBanks); + hipofileiterator.def_property_readonly("index", &HipoFileIterator::getIndex); + hipofileiterator.def_property_readonly("filenames", + &HipoFileIterator::getFilenames); + hipofileiterator.def_property_readonly("banknames", + &HipoFileIterator::getBanknames); + hipofileiterator.def_property_readonly("banks", + &HipoFileIterator::getBanklist); + hipofileiterator.def_property_readonly("types", + &HipoFileIterator::getTypelist); + hipofileiterator.def_property_readonly("items", + &HipoFileIterator::getItemlist); + hipofileiterator.def_property_readonly("reader", + &HipoFileIterator::getReader); + hipofileiterator.def_property_readonly("dict", &HipoFileIterator::getDict); + hipofileiterator.def_property_readonly("event", &HipoFileIterator::getEvent); + hipofileiterator.def_property_readonly("tags", &HipoFileIterator::getTags); + + hipofileiterator.def("__repr__", // TODO: Test this function in python + [](HipoFileIterator &hfi) { + std::string r("HipoFileIterator"); + return r; + }); + hipofileiterator.def("__eq__", // TODO: Test this function in python + [](HipoFileIterator *hfi1, HipoFileIterator *hfi2) { + return hfi1 == hfi2; + }); + hipofileiterator.def("__next__", // TODO: Test this function in python + [](HipoFileIterator &hfi) { + hfi.reset(); + return hfi.next(); + }); + + //----------------------------------------------------------------------// + // Bind HIPO utils + py::class_ utils(m, "Utils"); + utils.def(py::init<>()); + + utils.def("tokenize", &hipo::utils::tokenize); + utils.def("substring", &hipo::utils::substring); + utils.def("findposition", &hipo::utils::findposition); + utils.def("ltrim", &hipo::utils::ltrim); + utils.def("rtrim", &hipo::utils::rtrim); + utils.def("trim", &hipo::utils::trim); + + utils.def("printLogo", &hipo::utils::printLogo); + + utils.def("getHeader", &hipo::utils::getHeader); + utils.def("getFileHeader", &hipo::utils::getFileHeader); + utils.def("getFileTrailer", &hipo::utils::getFileTrailer); + utils.def("getSConstruct", &hipo::utils::getSConstruct); + + utils.def("writeInt", &hipo::utils::writeInt); + utils.def("writeLong", &hipo::utils::writeLong); + utils.def("writeByte", &hipo::utils::writeByte); + + //----------------------------------------------------------------------// + // Bind HIPO benchmark + py::class_ benchmark(m, "Benchmark"); + benchmark.def(py::init<>()); + benchmark.def( + py::init([](const char *name) { return new hipo::benchmark(name); })); + benchmark.def(py::init([](int freq) { return new hipo::benchmark(freq); })); + + benchmark.def("setName", &hipo::benchmark::setName); + benchmark.def("resume", &hipo::benchmark::resume); + benchmark.def("pause", &hipo::benchmark::pause); + benchmark.def("getTime", &hipo::benchmark::getTime); + benchmark.def("getTimeSec", &hipo::benchmark::getTimeSec); + benchmark.def("getCounter", &hipo::benchmark::getCounter); + benchmark.def("show", &hipo::benchmark::show); + + //----------------------------------------------------------------------// + // Bind HIPO schema + py::class_> schema(m, "Schema"); + schema.def(py::init<>()); + schema.def(py::init([](const char *name, int __groupid, int __itemid) { + return new hipo::schema(name, __groupid, __itemid); + })); + schema.def( + py::init([](const hipo::schema &s) { return new hipo::schema(s); })); + + schema.def("parse", &hipo::schema::parse); + schema.def("getName", &hipo::schema::getName); + schema.def("getGroup", &hipo::schema::getGroup); + schema.def("getItem", &hipo::schema::getItem); + schema.def("getSizeForRows", &hipo::schema::getSizeForRows); + schema.def("getRowLength", &hipo::schema::getRowLength); + schema.def("getEntryOrder", &hipo::schema::getEntryOrder); + schema.def("exists", &hipo::schema::exists); + schema.def("getOffset", + py::detail::overload_cast_impl()( + &hipo::schema::getOffset, py::const_), + "getOffset"); // NOTE: Need this syntax for const functions! + schema.def("getOffset", + py::detail::overload_cast_impl()( + &hipo::schema::getOffset, py::const_), + "getOffset"); // NOTE: Need this syntax for const functions! + schema.def("getEntryType", + py::detail::overload_cast_impl()(&hipo::schema::getEntryType, + py::const_), + "getEntryType"); // NOTE: Need this syntax for const functions! + schema.def("getEntryType", + py::detail::overload_cast_impl()( + &hipo::schema::getEntryType, py::const_), + "getEntryType"); // NOTE: Need this syntax for const functions! + schema.def("getEntryName", &hipo::schema::getEntryName); + schema.def("getSchemaString", &hipo::schema::getSchemaString); + schema.def("getSchemaStringJson", &hipo::schema::getSchemaStringJson); + + schema.def_property_readonly("schemaEntriesMap", nullptr); + schema.def_property_readonly("schemaEntries", nullptr); + schema.def_property_readonly("groupid", &hipo::schema::getGroup); + schema.def_property_readonly("itemid", &hipo::schema::getItem); + schema.def_property_readonly("rowLength", &hipo::schema::getRowLength); + schema.def_property_readonly("warningCount", nullptr); + schema.def_property_readonly("schemaName", &hipo::schema::getName); + + schema.def("__repr__", // TODO: Test this function in python + [](hipo::schema &s) { + std::string r("Schema : name = " + s.getName() + + " , schemaString = " + s.getSchemaString() + "\n"); + return r; + }); + schema.def("__eq__", // TODO: Test this function in python + [](hipo::schema *s1, hipo::schema *s2) { return s1 == s2; }); + schema.def("__len__", // TODO: Test this function in python + [](hipo::schema &s) { return s.getRowLength(); }); + + //----------------------------------------------------------------------// + // Bind HIPO dictionary + py::class_> dictionary(m, "Dictionary"); + dictionary.def(py::init<>()); + + dictionary.def("getSchemaList", &hipo::dictionary::getSchemaList); + dictionary.def("addSchema", &hipo::dictionary::addSchema); + dictionary.def("hasSchema", &hipo::dictionary::hasSchema); + dictionary.def("getSchema", &hipo::dictionary::getSchema); + dictionary.def("parse", &hipo::dictionary::parse); + dictionary.def("show", &hipo::dictionary::show); + + // schema.def_property_readonly("factory", nullptr); + + dictionary.def("__repr__", // TODO: Test this function in python + [](hipo::dictionary &d) { + std::vector schemaList = d.getSchemaList(); + std::string r("Dictionary :\n"); + int schemalist_size = schemaList.size(); + for (int idx = 0; idx < schemalist_size; idx++) { + const char *buffer = schemaList[idx].c_str(); + hipo::schema s = d.getSchema(buffer); + r += "\tSchema : name = " + s.getName() + + " , schemaString = " + s.getSchemaString() + "\n"; + } + return r; + }); + dictionary.def( + "__eq__", // TODO: Test this function in python + [](hipo::dictionary *d1, hipo::dictionary *d2) { return d1 == d2; }); + dictionary.def("__len__", // TODO: Test this function in python + [](hipo::dictionary &d) { return d.getSchemaList().size(); }); + + //----------------------------------------------------------------------// + // Bind HIPO structure + py::class_> structure(m, "Structure"); + structure.def(py::init<>()); + structure.def(py::init([](int size) { return new hipo::structure(size); })); + structure.def(py::init([](int __group, int __item, std::string &str) { + return new hipo::structure(__group, __item, str); + })); + + structure.def("allocate", &hipo::structure::allocate); + structure.def("getSize", &hipo::structure::getSize); + structure.def("getType", &hipo::structure::getType); + structure.def("getGroup", &hipo::structure::getGroup); + structure.def("getItem", &hipo::structure::getItem); + structure.def("init", &hipo::structure::init); + structure.def("initNoCopy", &hipo::structure::initNoCopy); + structure.def("getAddress", &hipo::structure::getAddress); + structure.def("show", &hipo::structure::show); + structure.def("setSize", &hipo::structure::setSize); + structure.def("getIntAt", &hipo::structure::getIntAt); + structure.def("getShortAt", &hipo::structure::getShortAt); + structure.def("getByteAt", &hipo::structure::getByteAt); + structure.def("getFloatAt", &hipo::structure::getFloatAt); + structure.def("getDoubleAt", &hipo::structure::getDoubleAt); + structure.def("getLongAt", &hipo::structure::getLongAt); + structure.def("getStringAt", &hipo::structure::getStringAt); + structure.def("putIntAt", &hipo::structure::putIntAt); + structure.def("putShortAt", &hipo::structure::putShortAt); + structure.def("putByteAt", &hipo::structure::putByteAt); + structure.def("putFloatAt", &hipo::structure::putFloatAt); + structure.def("putDoubleAt", &hipo::structure::putDoubleAt); + structure.def("putLongAt", &hipo::structure::putLongAt); + structure.def("putStringAt", &hipo::structure::putStringAt); + structure.def("notify", &hipo::structure::notify); + + structure.def("__repr__", // TODO: Test this function in python + [](hipo::structure &s) { + std::string r("Structure : group = "); + r += std::to_string(s.getGroup()) + + " , item = " + std::to_string(s.getItem()) + + " , type = " + std::to_string(s.getType()) + + " , length = " + std::to_string(s.getSize()) + "\n"; + return r; + }); + structure.def( + "__eq__", // TODO: Test this function in python + [](hipo::structure *s1, hipo::structure *s2) { return s1 == s2; }); + structure.def("__len__", // TODO: Test this function in python + [](hipo::structure &s) { return s.getSize(); }); + + //----------------------------------------------------------------------// + // Bind HIPO bank + py::class_> bank(m, "Bank"); + bank.def(py::init<>()); + bank.def(py::init( + [](const hipo::schema &__schema) { return new hipo::bank(__schema); })); + bank.def(py::init([](const hipo::schema &__schema, int __rows) { + return new hipo::bank(__schema, __rows); + })); + + bank.def("getSchema", &hipo::bank::getSchema); + bank.def("getRows", &hipo::bank::getRows); + bank.def("setRows", &hipo::bank::setRows); + + bank.def("getInt", + py::detail::overload_cast_impl()(&hipo::bank::getInt, + py::const_), + "getInt"); + bank.def("getInt", + py::detail::overload_cast_impl()( + &hipo::bank::getInt, py::const_), + "getInt"); + bank.def("getShort", + py::detail::overload_cast_impl()(&hipo::bank::getShort, + py::const_), + "getShort"); + bank.def("getShort", + py::detail::overload_cast_impl()( + &hipo::bank::getShort, py::const_), + "getShort"); + bank.def("getByte", + py::detail::overload_cast_impl()(&hipo::bank::getByte, + py::const_), + "getByte"); + bank.def("getByte", + py::detail::overload_cast_impl()( + &hipo::bank::getByte, py::const_), + "getByte"); + bank.def("getFloat", + py::detail::overload_cast_impl()(&hipo::bank::getFloat, + py::const_), + "getFloat"); + bank.def("getFloat", + py::detail::overload_cast_impl()( + &hipo::bank::getFloat, py::const_), + "getFloat"); + bank.def("getDouble", + py::detail::overload_cast_impl()(&hipo::bank::getDouble, + py::const_), + "getDouble"); + bank.def("getDouble", + py::detail::overload_cast_impl()( + &hipo::bank::getDouble, py::const_), + "getDouble"); + bank.def("getLong", + py::detail::overload_cast_impl()(&hipo::bank::getLong, + py::const_), + "getLong"); + bank.def("getLong", + py::detail::overload_cast_impl()( + &hipo::bank::getLong, py::const_), + "getLong"); + + bank.def( + "putInt", + py::detail::overload_cast_impl()(&hipo::bank::putInt), + "putInt"); + bank.def("putInt", + py::detail::overload_cast_impl()( + &hipo::bank::putInt), + "putInt"); + bank.def("putShort", + py::detail::overload_cast_impl()( + &hipo::bank::putShort), + "putShort"); + bank.def("putShort", + py::detail::overload_cast_impl()( + &hipo::bank::putShort), + "putShort"); + bank.def( + "putByte", + py::detail::overload_cast_impl()(&hipo::bank::putByte), + "putByte"); + bank.def("putByte", + py::detail::overload_cast_impl()( + &hipo::bank::putByte), + "putByte"); + bank.def( + "putFloat", + py::detail::overload_cast_impl()(&hipo::bank::putFloat), + "putFloat"); + bank.def("putFloat", + py::detail::overload_cast_impl()( + &hipo::bank::putFloat), + "putFloat"); + bank.def("putDouble", + py::detail::overload_cast_impl()( + &hipo::bank::putDouble), + "putDouble"); + bank.def("putDouble", + py::detail::overload_cast_impl()( + &hipo::bank::putDouble), + "putDouble"); + bank.def( + "putLong", + py::detail::overload_cast_impl()(&hipo::bank::putLong), + "putLong"); + bank.def("putLong", + py::detail::overload_cast_impl()( + &hipo::bank::putLong), + "putLong"); + + bank.def("getRowList", &hipo::bank::getRowList); + bank.def("getFullRowList", &hipo::bank::getFullRowList); + bank.def("getMutableRowList", &hipo::bank::getMutableRowList); + bank.def("getRowListLinked", &hipo::bank::getRowListLinked); + + bank.def("show", + py::detail::overload_cast_impl<>()(&hipo::bank::show, py::const_), + "show"); // NOTE: Need this syntax for const functions! + bank.def( + "show", + py::detail::overload_cast_impl()(&hipo::bank::show, py::const_), + "show"); // NOTE: Need this syntax for const functions! + bank.def("printValue", &hipo::bank::printValue); + bank.def("reset", &hipo::bank::reset); + bank.def("notify", &hipo::bank::notify); + + bank.def_property_readonly( + "bankSchema", &hipo::bank::getSchema); // NOTE: Not really necessary. + bank.def_property_readonly("bankRows", &hipo::bank::getRows); + + bank.def( + "__repr__", // TODO: Test this function in python + [](hipo::bank &b) { + std::string r("Bank : name = "); + r += b.getSchema().getName() + + " , rows = " + std::to_string(b.getRows()) + "\n"; + for (int i = 0; i < b.getSchema().getEntries(); i++) { + std::string x("\t"); + x += b.getSchema().getEntryName(i) + " : "; + r += x; + for (int k = 0; k < b.getRows(); k++) { + if (b.getSchema().getEntryType(i) < 4) { + r += std::to_string(b.getInt(i, k)) + + " "; // std::string y("%8d ", b.getInt(i,k)); r += y; // %8d + } else { + if (b.getSchema().getEntryType(i) == 4) { + r += std::to_string(b.getFloat(i, k)) + + " "; // std::string y("%8.5f ",b.getFloat(i,k)); r += y; // + // %8.5f + } + if (b.getSchema().getEntryType(i) == 5) { + r += std::to_string(b.getDouble(i, k)) + + " "; // std::string y("%8.5f ",b.getDouble(i,k)); r += y; + // // %8.5f + } + if (b.getSchema().getEntryType(i) == 8) { + r += std::to_string(b.getLong(i, k)) + + " "; // std::string y("%14ld ", b.getLong(i,k)); r += y; // + // %14ld + } + } // else + } + r += "\n"; + } // for(int i = 0; i < b.getSchema().getEntries(); i++) + return r; + } // [](hipo::bank &b) + ); + bank.def("__eq__", // TODO: Test this function in python + [](hipo::bank *b1, hipo::bank *b2) { return b1 == b2; }); + bank.def("__len__", // TODO: Test this function in python + [](hipo::bank &b) { return b.getRows(); }); + + //----------------------------------------------------------------------// + // Bind HIPO event + py::class_> event(m, "Event"); + event.def(py::init<>()); + event.def(py::init([](int size) { return new hipo::event(size); })); + + event.def("show", &hipo::event::show); + event.def("init", + static_cast &)>( + &hipo::event::init), + "init"); + event.def( + "init", + static_cast(&hipo::event::init), + "init"); + event.def("getStructure", + static_cast( + &hipo::event::getStructure), + "getStructure"); + event.def("getStructure4", &hipo::event::getStructure4); + + event.def("getTag", &hipo::event::getTag); + event.def("setTag", &hipo::event::setTag); + event.def("getStructure", + static_cast( + &hipo::event::getStructure), + "getStructure"); + event.def( + "read", + static_cast(&hipo::event::read), + "read"); + event.def("addStructure", &hipo::event::addStructure); + event.def("override", &hipo::event::override); + event.def( + "remove", + static_cast(&hipo::event::remove), + "remove"); + event.def("remove", + static_cast(&hipo::event::remove), + "remove"); + event.def("replace", &hipo::event::replace); + + event.def("add", &hipo::event::add); + event.def("get", + static_cast( + &hipo::event::get), + "get"); + + event.def("getStructurePosition", + static_cast (hipo::event::*)(int, int)>( + &hipo::event::getStructurePosition), + "getStructurePosition"); + event.def("getStructurePosition4", + static_cast (hipo::event::*)(int, int)>( + &hipo::event::getStructurePosition4), + "getStructurePosition4"); + + event.def("getEventBuffer", &hipo::event::getEventBuffer); + event.def("getSize", &hipo::event::getSize); + event.def("reset", &hipo::event::reset); + event.def("write", &hipo::event::write); + event.def("read", + static_cast( + &hipo::event::read), + "read"); + + event.def_static("getStructurePosition_static", + static_cast (*)(const char *, int, int)>( + &hipo::event::getStructurePosition), + "getStructurePosition"); + event.def_static( + "getStructure_static", + static_cast( + &hipo::event::getStructure), + "getStructure"); + event.def_static("get_static", + static_cast( + &hipo::event::get), + "get"); + event.def_static("getStructureNoCopy", &hipo::event::getStructureNoCopy); + + // event.def_property_readonly("databuffer", nullptr); //NOTE: Not really + // necessary. + + event.def( + "__repr__", // TODO: Test this function in python + [](hipo::event &e) { + std::string r("Event : size = "); + r += std::to_string(e.getSize()) + "\n"; + int position = 16; + int eventSize = *(reinterpret_cast(&e.getEventBuffer()[4])); + while (position + 8 < eventSize) { + uint16_t gid = + *(reinterpret_cast(&e.getEventBuffer()[position])); + uint8_t iid = + *(reinterpret_cast(&e.getEventBuffer()[position + 2])); + uint8_t type = + *(reinterpret_cast(&e.getEventBuffer()[position + 3])); + int length = + *(reinterpret_cast(&e.getEventBuffer()[position + 4])); + r += "\tgroup = " + std::to_string(gid) + + " , item = " + std::to_string(iid) + + " , type = " + std::to_string(type) + + " , length = " + std::to_string(length) + "\n"; + position += (length + 8); + } + return r; + } // [](hipo::event &e) + ); + event.def("__eq__", // TODO: Test this function in python + [](hipo::event *e1, hipo::event *e2) { return e1 == e2; }); + event.def("__len__", // TODO: Test this function in python + [](hipo::event &e) { return e.getSize(); }); + + //----------------------------------------------------------------------// + // Bind HIPO readerIndex + py::class_ readerIndex( + m, "ReaderIndex"); // NOTE: Can't use trampoline class because you get + // this error: "Cannot use an alias class with a + // non-polymorphic type" + readerIndex.def(py::init<>([]() { return new hipo::readerIndex(); })); + + readerIndex.def("canAdvance", &hipo::readerIndex::canAdvance); + readerIndex.def("advance", &hipo::readerIndex::advance); + + readerIndex.def("canAdvanceInRecord", &hipo::readerIndex::canAdvanceInRecord); + readerIndex.def("loadRecord", &hipo::readerIndex::loadRecord); + readerIndex.def("gotoEvent", &hipo::readerIndex::gotoEvent); + readerIndex.def("gotoRecord", &hipo::readerIndex::gotoRecord); + + readerIndex.def("getEventNumber", &hipo::readerIndex::getEventNumber); + readerIndex.def("getRecordNumber", &hipo::readerIndex::getRecordNumber); + readerIndex.def("getRecordEventNumber", + &hipo::readerIndex::getRecordEventNumber); + readerIndex.def("getMaxEvents", &hipo::readerIndex::getMaxEvents); + readerIndex.def("addSize", &hipo::readerIndex::addSize); + readerIndex.def("addPosition", &hipo::readerIndex::addPosition); + readerIndex.def("getPosition", &hipo::readerIndex::getPosition); + + readerIndex.def("getNRecords", &hipo::readerIndex::getNRecords); + readerIndex.def("rewind", &hipo::readerIndex::rewind); + readerIndex.def("clear", &hipo::readerIndex::clear); + readerIndex.def("reset", &hipo::readerIndex::reset); + + // readerIndex.def_property_readonly("recordEvents", nullptr); //NOTE: Not + // really necessary. readerIndex.def_property_readonly("recordPosition", + // nullptr); + readerIndex.def_property_readonly("currentRecord", + &hipo::readerIndex::getRecordNumber); + readerIndex.def_property_readonly("currentEvent", + &hipo::readerIndex::getEventNumber); + readerIndex.def_property_readonly("currentRecordEvent", + &hipo::readerIndex::getRecordEventNumber); + + readerIndex.def("__repr__", [](hipo::readerIndex &r) { + std::string x("Reader Index : "); + x += "nrecords = " + std::to_string(r.getNRecords()) + + " , record number = " + std::to_string(r.getRecordNumber()) + + " , event number = " + std::to_string(r.getEventNumber()) + "\n"; + return x; + }); + + readerIndex.def( + "__eq__", // TODO: Test this function in python + [](hipo::readerIndex *r1, hipo::readerIndex *r2) { return r1 == r2; }); + readerIndex.def("__len__", // TODO: Test this function in python + [](hipo::readerIndex &r) { return r.getNRecords(); }); + + //----------------------------------------------------------------------// + // Bind HIPO reader + py::class_ reader(m, "Reader"); + reader.def(py::init<>([]() { return new hipo::reader(); })); + reader.def( + py::init([](const hipo::reader &r) { return new hipo::reader(r); })); + + reader.def("about", &hipo::reader::about); + reader.def("rewind", &hipo::reader::rewind); + reader.def("readDictionary", &hipo::reader::readDictionary); + reader.def("getStructure", &hipo::reader::getStructure); + reader.def("getStructureNoCopy", &hipo::reader::getStructureNoCopy); + + reader.def("readUserConfig", &hipo::reader::readUserConfig); + + reader.def("open", &hipo::reader::open); + reader.def("is_open", &hipo::reader::is_open); + reader.def("setTags", + static_cast(&hipo::reader::setTags), + "setTags"); + reader.def("setTags", + static_cast)>( + &hipo::reader::setTags), + "setTags"); + reader.def("setVerbose", &hipo::reader::setVerbose); + + reader.def("hasNext", &hipo::reader::hasNext); + reader.def("next", static_cast(&hipo::reader::next), + "next"); + reader.def("gotoEvent", &hipo::reader::gotoEvent); + reader.def("gotoRecord", &hipo::reader::gotoRecord); + reader.def( + "next", + static_cast(&hipo::reader::next), + "next"); + + reader.def("next", + static_cast &)>( + &hipo::reader::next), + "next"); + reader.def("getBanks", &hipo::reader::getBanks); + + reader.def("read", &hipo::reader::read); + reader.def("printWarning", &hipo::reader::printWarning); + + reader.def("getNRecords", &hipo::reader::getNRecords); + reader.def("nextInRecord", &hipo::reader::nextInRecord); + reader.def( + "loadRecord", + static_cast(&hipo::reader::loadRecord), + "loadRecord"); + reader.def("loadRecord", + static_cast( + &hipo::reader::loadRecord), + "loadRecord"); + reader.def("getEntries", &hipo::reader::getEntries); + reader.def("getInt", &hipo::reader::getInt); + reader.def("getFloat", &hipo::reader::getFloat); + + reader.def("__repr__", [](hipo::reader &r) { + std::string x("Reader : "); + x += "nrecords = " + std::to_string(r.getNRecords()) + + " , entries = " + std::to_string(r.getEntries()) + "\n"; + return x; + }); + reader.def("__eq__", // TODO: Test this function in python + [](hipo::reader *r1, hipo::reader *r2) { return r1 == r2; }); + reader.def("__len__", // TODO: Test this function in python + [](hipo::reader &r) { return r.getNRecords(); }); + + //----------------------------------------------------------------------// + // Bind HIPO data + py::class_ data(m, "Data"); + data.def(py::init<>()); + + data.def("setDataPtr", &hipo::data::setDataPtr); + data.def("setDataSize", &hipo::data::setDataSize); + data.def("setDataOffset", &hipo::data::setDataOffset); + data.def("setDataEndianness", &hipo::data::setDataEndianness); + + data.def("getEvioPtr", &hipo::data::getEvioPtr); + data.def("getEvioSize", &hipo::data::getEvioSize); + data.def("getDataPtr", &hipo::data::getDataPtr); + data.def("getDataSize", &hipo::data::getDataSize); + data.def("getDataOffset", &hipo::data::getDataOffset); + data.def("getDataEndianness", &hipo::data::getDataEndianness); + + data.def_property_readonly("data_ptr", &hipo::data::getDataPtr); + data.def_property_readonly("data_size", &hipo::data::getDataSize); + data.def_property_readonly("data_endianness", &hipo::data::getDataEndianness); + data.def_property_readonly("data_offset", &hipo::data::getDataOffset); + + data.def("__repr__", [](hipo::data &d) { + std::string r("Data : "); + r += "size = " + std::to_string(d.getDataSize()) + "\n"; + return r; + }); + data.def("__eq__", // TODO: Test this function in python + [](hipo::data *d1, hipo::data *d2) { return d1 == d2; }); + data.def("__len__", // TODO: Test this function in python + [](hipo::data &d) { return d.getDataSize(); }); + + //----------------------------------------------------------------------// + // Bind HIPO record + py::class_ record(m, "Record"); + record.def(py::init<>()); + + record.def("readRecord", + static_cast( + &hipo::record::readRecord), + "readRecord"); + record.def( + "readRecord", + static_cast( + &hipo::record::readRecord), + "readRecord"); + record.def("readRecord__", &hipo::record::readRecord__); + record.def("getEventCount", &hipo::record::getEventCount); + record.def("getRecordSizeCompressed", &hipo::record::getRecordSizeCompressed); + + record.def("readEvent", &hipo::record::readEvent); + record.def("readHipoEvent", &hipo::record::readHipoEvent); + record.def("getData", &hipo::record::getData); + + record.def("getReadBenchmark", &hipo::record::getReadBenchmark); + record.def("getUnzipBenchmark", &hipo::record::getUnzipBenchmark); + record.def("getIndexBenchmark", &hipo::record::getIndexBenchmark); + + record.def_property_readonly("readBenchmark", + &hipo::record::getReadBenchmark); + record.def_property_readonly("unzipBenchmark", + &hipo::record::getUnzipBenchmark); + record.def_property_readonly("indexBenchmark", + &hipo::record::getIndexBenchmark); + + record.def("__repr__", [](hipo::record &r) { + std::string x("Record : "); + x += "event count = " + std::to_string(r.getEventCount()) + "\n"; + return x; + }); + record.def("__eq__", // TODO: Test this function in python + [](hipo::record *r1, hipo::record *r2) { return r1 == r2; }); + record.def("__len__", // TODO: Test this function in python + [](hipo::record &r) { return r.getEventCount(); }); + + //----------------------------------------------------------------------// + // Bind HIPO recordbuilder + py::class_> recordbuilder( + m, "RecordBuilder"); + recordbuilder.def(py::init<>([]() { return new hipo::recordbuilder(); })); + recordbuilder.def(py::init<>([](int maxEvents, int maxLength) { + return new hipo::recordbuilder(maxEvents, maxLength); + })); + + recordbuilder.def("addEvent", + static_cast( + &hipo::recordbuilder::addEvent), + "addEvent"); + recordbuilder.def( + "addEvent", + static_cast &, int, int)>( + &hipo::recordbuilder::addEvent), + "addEvent"); + + recordbuilder.def("getUserWordOne", &hipo::recordbuilder::getUserWordOne); + recordbuilder.def("getUserWordTwo", &hipo::recordbuilder::getUserWordTwo); + recordbuilder.def("setUserWordOne", &hipo::recordbuilder::setUserWordOne); + recordbuilder.def("setUserWordTwo", &hipo::recordbuilder::setUserWordTwo); + + recordbuilder.def("getRecordSize", &hipo::recordbuilder::getRecordSize); + recordbuilder.def("getEntries", &hipo::recordbuilder::getEntries); + recordbuilder.def("getRecordBuffer", &hipo::recordbuilder::getRecordBuffer); + recordbuilder.def("reset", &hipo::recordbuilder::reset); + recordbuilder.def("build", &hipo::recordbuilder::build); + + recordbuilder.def_property_readonly("bufferUserWordOne", + &hipo::recordbuilder::getUserWordOne); + recordbuilder.def_property_readonly("bufferUserWordTwo", + &hipo::recordbuilder::getUserWordTwo); + + recordbuilder.def("__repr__", [](hipo::recordbuilder &r) { + std::string x("Recordbuilder : "); + x += "entries = " + std::to_string(r.getEntries()) + + " , userWordOne = " + std::to_string(r.getUserWordOne()) + + " , userWordTwo = " + std::to_string(r.getUserWordTwo()) + "\n"; + return x; + }); + recordbuilder.def("__eq__", // TODO: Test this function in python + [](hipo::recordbuilder *r1, hipo::recordbuilder *r2) { + return r1 == r2; + }); + recordbuilder.def("__len__", // TODO: Test this function in python + [](hipo::recordbuilder &r) { return r.getEntries(); }); + + //----------------------------------------------------------------------// + // Bind HIPO writer + py::class_> writer(m, "Writer"); + writer.def(py::init<>([]() { return new hipo::writer(); })); + + writer.def("addEvent", + static_cast( + &hipo::writer::addEvent), + "addEvent"); + writer.def("addEvent", + static_cast &, int)>( + &hipo::writer::addEvent), + "addEvent"); + writer.def("writeRecord", &hipo::writer::writeRecord); + writer.def("open", &hipo::writer::open); + writer.def("close", &hipo::writer::close); + writer.def("showSummary", &hipo::writer::showSummary); + writer.def("addDictionary", &hipo::writer::addDictionary); + writer.def("getDictionary", &hipo::writer::getDictionary); + writer.def("setUserIntegerOne", &hipo::writer::setUserIntegerOne); + writer.def("setUserIntegerTwo", &hipo::writer::setUserIntegerTwo); + writer.def("flush", &hipo::writer::flush); + + writer.def_property_readonly("writerDictionary", + &hipo::writer::getDictionary); + + writer.def("__repr__", [](hipo::writer &w) { + std::string r("Writer : "); + return r + "\n"; + }); + writer.def("__eq__", // TODO: Test this function in python + [](hipo::writer *w1, hipo::writer *w2) { return w1 == w2; }); + + //----------------------------------------------------------------------// + // Versioning + +#ifdef VERSION_INFO + m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); +#else + m.attr("__version__") = "dev"; +#endif +} // PYBIND11_MODULE diff --git a/src/main.cpp b/src/main.cpp deleted file mode 100644 index 0263112..0000000 --- a/src/main.cpp +++ /dev/null @@ -1,1097 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define STRINGIFY(x) #x -#define MACRO_STRINGIFY(x) STRINGIFY(x) - -//NOTE: Shared object will not load if you simultaneously -// define classes to bind within this file and bind hipo -// classes or classes from another cpp project. - -//--------------------------------------------------// -// Custom classes - -class HipoFileIterator { - - private: - int batchsize; - int nbanks; - int index; - std::vector filenames; - std::vector banknames; - std::vector banklist; - std::vector> typelist; - std::vector> itemlist; - hipo::reader reader; - // hipo::writer writer; - hipo::dictionary dict; - hipo::event event; - - std::vector>> vec_double; - std::vector>> vec_float; - std::vector>> vec_int; - std::vector>> vec_long; - std::vector>> vec_short; - std::vector>> vec_byte; - - std::map item_index_map; - std::map item_type_map; - std::string separator; - std::vector item_indices; - std::vector tags; - - protected: - void protected_method(); - - public: - HipoFileIterator( - std::vector &__filenames__, - std::vector &__banknames__, - int __batchsize__, - std::vector &__tags__ - ){ - filenames = __filenames__; - banknames = __banknames__; - nbanks = banknames.size(); - batchsize = __batchsize__; - separator = "_"; - item_indices = std::vector(9); //NOTE: Only 6 types, but they only go up to 8 and it's easier to use the type int as the index. - tags = __tags__; - - // Check length of filenames - if (filenames.size()==0) throw std::invalid_argument("HipoFileIterator: length of filenames must be > 0"); - - // Check batchsize - if (batchsize<=0) throw std::invalid_argument("HipoFileIterator: batch_size must be > 0"); - - index = -1; - - // Set tags before opening files - for (int i=0; i>(0); - itemlist = std::vector>(0); - for (int i = 0; i vec_entrytype = std::vector(0); - std::vector vec_entryname = std::vector(0); //NOTE: IMPORTANT THAT THIS IS ZERO. IF YOU SET TO NENTRIES YOU MESS UP PUSHBACK AND GET A VECTOR TWICE AS LONG AS YOU WANT. - for (int j = 0; j=filenames.size()) return false; - const char * filename = filenames.at(index).c_str(); - reader.open(filename); - reader.readDictionary(dict); //TODO: Figure out how to check file here.... - return true; - } - - /** - * Resets the vectors used for storing batch data of all available types. - * @exception std::out_of_range - */ - void reset() { - try { - vec_double = std::vector>>(item_indices.at(5)); - vec_float = std::vector>>(item_indices.at(4)); - vec_int = std::vector>>(item_indices.at(3)); - vec_long = std::vector>>(item_indices.at(8)); - vec_short = std::vector>>(item_indices.at(2)); - vec_byte = std::vector>>(item_indices.at(1)); - } catch(const std::out_of_range &e) { - std::cout << e.what() < tags - */ - std::vector getTags() { - return tags; - } - - /** - * Recursive function which loops file(s) to read data into next batch. - * @param int __counter__ - * @return bool - */ - bool next(int __counter__ = 0) { - - int counter = __counter__; - - // Loop events - while (reader.next()) { - reader.read(event); - - // Loop banks - for (int i = 0; i entrytypes = typelist.at(i); - std::vector entrynames = itemlist.at(i); //TODO: COULD ALSO COMPARE TIME WITH SCHEMALIST... - int nentries = entrytypes.size(); - - // Loop entries - for (int j = 0; j GetDoubles etc should return empty array if bank is empty in event and schema is still there so this should be fine, in principle. - case 5: vec_double.at(entryindex).push_back(bank.getDoubles(name)); break; - case 8: vec_long.at(entryindex).push_back(bank.getLongs(name)); break; - default: throw pybind11::type_error("HipoFileIterator: Invalid type int for entry: "+entrynames.at(j)+"\n"); - } - } // for (int j = 0; j=batchsize) { - if (!reader.hasNext() && index>=filenames.size()-1) return false; - return true; - } - } - // Move on to next file if batch is not yet complete - if (open()) { - return next(counter); //NOTE: Recursive function //NOTE: If you do not return here, the original function keeps going after the recursive call and returns false and so ends iteration. - } - //TODO: NOTE: THIS SHOULD NOT RAISE A STOP ITERATION BECAUSE YOU STILL NEED TO GET THE BATCH DATA IN PYTHON - return false; //NOTE: Only happens if you've reached the end of all files. - } - - /** - * Get type int for given bank name and entry name. - * @param std::string bankname - * @param std::string item - * @return int i - */ - int getType(std::string bankname, std::string item) { - return item_type_map.at(bankname+separator+item); //NOTE: //TODO: COULD ADD pybind11::attribute_error here and try block. - } - - /** - * Get batch array of doubles for given bank name and entry name. - * @param std::string bankname - * @param std::string item - * @return std::vector> - */ - std::vector> getDoubles(std::string bankname, std::string item) const noexcept { //TODO: Check speed with these methods as inline. - int i = item_index_map.at(bankname+separator+item); - return vec_double.at(i); - } - - /** - * Get batch array of floats for given bank name and entry name. - * @param std::string bankname - * @param std::string item - * @return std::vector> - */ - std::vector> getFloats(std::string bankname, std::string item) const noexcept { - int i = item_index_map.at(bankname+separator+item); - return vec_float.at(i); - } - - /** - * Get batch array of ints for given bank name and entry name. - * @param std::string bankname - * @param std::string item - * @return std::vector> - */ - std::vector> getInts(std::string bankname, std::string item) const noexcept { - int i = item_index_map.at(bankname+separator+item); - return vec_int.at(i); - } - - /** - * Get batch array of longs for given bank name and entry name. - * @param std::string bankname - * @param std::string item - * @return std::vector> - */ - std::vector> getLongs(std::string bankname, std::string item) const noexcept { - int i = item_index_map.at(bankname+separator+item); - return vec_long.at(i); - } - - /** - * Get batch array of shorts for given bank name and entry name. - * @param std::string bankname - * @param std::string item - * @return std::vector> - */ - std::vector> getShorts(std::string bankname, std::string item) const noexcept { - int i = item_index_map.at(bankname+separator+item); - return vec_short.at(i); - } - - /** - * Get batch array of bytes for given bank name and entry name. - * @param std::string bankname - * @param std::string item - * @return std::vector> - */ - std::vector> getBytes(std::string bankname, std::string item) const noexcept { - int i = item_index_map.at(bankname+separator+item); - return vec_byte.at(i); - } - - /* - * Get methods for class attributes - */ - int getBatchsize() { return batchsize; } - int getNBanks() { return nbanks; } - int getIndex() { return index; } - std::vector getFilenames() { return filenames; } - std::vector getBanknames() { return banknames; } - std::vector getBanklist() { return banklist; } - std::vector> getTypelist() { return typelist; } - std::vector> getItemlist() { return itemlist; } - hipo::reader getReader() { return reader; } - // hipo::writer getWriter() { return writer; } - hipo::dictionary getDict() { return dict; } - hipo::event getEvent() { return event; } - - }; // class HipoFileIterator - -//--------------------------------------------------// -// HipoFileIterator Trampoline class -template class PyHipoFileIterator : public HipoFileIteratorBase { -public: - using HipoFileIteratorBase::HipoFileIteratorBase; // Inherit constructors -}; - -//--------------------------------------------------// -// HIPO Trampoline classes - -//NOTE: utils, benchmark classes throw compilation error if you try to use trampoline class. - -template class PyDatastream : public DatastreamBase { -public: - using DatastreamBase::DatastreamBase; // Inherit constructors - long size() override { PYBIND11_OVERRIDE_PURE(long, DatastreamBase, size, ); } - long position() override { PYBIND11_OVERRIDE_PURE(long, DatastreamBase, position, ); } - long position(long pos) override { PYBIND11_OVERRIDE_PURE(long, DatastreamBase, position, pos); } - void open(const char *filename) override { PYBIND11_OVERRIDE_PURE(void, DatastreamBase, open, filename ); } - int read(char *s, int size) override { PYBIND11_OVERRIDE_PURE(int, DatastreamBase, read, s, size); } -}; - -//NOTE: datastreamLocalFile, datastreamXrootd classes thorw compilation error if you try to use trampoline class. - -template class PySchema : public SchemaBase { -public: - using SchemaBase::SchemaBase; // Inherit constructors -}; - -template class PyDictionary : public DictionaryBase { -public: - using DictionaryBase::DictionaryBase; // Inherit constructors -}; - -template class PyStructure : public StructureBase { -public: - using StructureBase::StructureBase; // Inherit constructors - void notify() override { PYBIND11_OVERRIDE_PURE(void, StructureBase, notify, ); } -}; - -template class PyBank : public PyStructure { -public: - using PyStructure::PyStructure; // Inherit constructors -}; - -template class PyEvent : public EventBase { -public: - using EventBase::EventBase; // Inherit constructors -}; - -//NOTE: readerIndex, reader, data, record classes throw compilation error if you try to use trampoline class. - -template class PyRecordBuilder : public RecordBuilderBase { -public: - using RecordBuilderBase::RecordBuilderBase; // Inherit constructors -}; - -template class PyWriter : public WriterBase { -public: - using WriterBase::WriterBase; // Inherit constructors -}; - -//--------------------------------------------------// -// Python Module Bindings - -namespace py = pybind11; -PYBIND11_MODULE(hipopybind, m) { - - //ADDED BEGIN - //----------------------------------------------------------------------// - // Bind HipoFileIterator - py::class_> hipofileiterator(m, "HipoFileIterator"); - hipofileiterator.def(py::init([](std::vector & filenames, std::vector & banknames, int batchsize, std::vector & tags) { return new HipoFileIterator(filenames,banknames,batchsize,tags); })); - hipofileiterator.def("open", &HipoFileIterator::open); - hipofileiterator.def("reset", &HipoFileIterator::reset); - hipofileiterator.def("next", &HipoFileIterator::next); - hipofileiterator.def("gotoEvent", &HipoFileIterator::gotoEvent); - hipofileiterator.def("setTags", &HipoFileIterator::setTags); //TODO: ADD OPTION IN INIT TO SET TAGS... - hipofileiterator.def("getType", &HipoFileIterator::getType); - hipofileiterator.def("getDoubles", &HipoFileIterator::getDoubles); - hipofileiterator.def("getFloats", &HipoFileIterator::getFloats); - hipofileiterator.def("getInts", &HipoFileIterator::getInts); - hipofileiterator.def("getLongs", &HipoFileIterator::getLongs); - hipofileiterator.def("getShorts", &HipoFileIterator::getShorts); - hipofileiterator.def("getBytes", &HipoFileIterator::getBytes); - - hipofileiterator.def_property_readonly("batchsize",&HipoFileIterator::getBatchsize); - hipofileiterator.def_property_readonly("nbanks",&HipoFileIterator::getNBanks); - hipofileiterator.def_property_readonly("index",&HipoFileIterator::getIndex); - hipofileiterator.def_property_readonly("filenames",&HipoFileIterator::getFilenames); - hipofileiterator.def_property_readonly("banknames",&HipoFileIterator::getBanknames); - hipofileiterator.def_property_readonly("banks",&HipoFileIterator::getBanklist); - hipofileiterator.def_property_readonly("types",&HipoFileIterator::getTypelist); - hipofileiterator.def_property_readonly("items",&HipoFileIterator::getItemlist); - hipofileiterator.def_property_readonly("reader",&HipoFileIterator::getReader); - hipofileiterator.def_property_readonly("dict",&HipoFileIterator::getDict); - hipofileiterator.def_property_readonly("event",&HipoFileIterator::getEvent); - hipofileiterator.def_property_readonly("tags",&HipoFileIterator::getTags); - - hipofileiterator.def("__repr__", //TODO: Test this function in python - [](HipoFileIterator &hfi) { std::string r("HipoFileIterator"); return r; } - ); - hipofileiterator.def("__eq__", //TODO: Test this function in python - [](HipoFileIterator *hfi1, HipoFileIterator *hfi2) { return hfi1 == hfi2; } - ); - hipofileiterator.def("__next__", //TODO: Test this function in python - [](HipoFileIterator &hfi) { hfi.reset(); return hfi.next(); } - ); - - // * DONE *: TODO: ADD METHOD TO GET TYPE OF KEY AND THEN GET ARRAY? -> Can then use __get__ method - // * DONE *: TODO: ADD RAISE STOP_ITERATION IN __NEXT__ - // * DONE *: TODO: ADD GET SET METHOD AND ADD def.property for accessible attributes like reader and banklist and filenames and banknames and entry names and batch size and index and so on.... - //TODO: HOW TO LINK AWKWARD / NUMPY ARRAYS DIRECTLY TO RETURNED VECTORS - //TODO: LONGTERM: HipoFileWriter -> Handles reading and writing simultaneously. - //TODO: LONGTERM: Writing to file with [] operator accessor. - - - // hipofileiterator.def("__get__", //NOTE: THIS WILL NOT WORK SINCE THE LAMBDA NEEDS TO HAVE THE SAME RETURN TYPE EVERY TIME... - // [](HipoFileIterator &hfi,std::string bankname, std::string item) { - // int type = hfi.getType(bankname,item); - // switch (type) { - // case 3: return hfi.getInts(bankname,item); - // case 4: return hfi.getFloats(bankname,item); - // case 5: return hfi.getDoubles(bankname,item); - // case 8: return hfi.getLongs(bankname,item); - // case 1: return hfi.getBytes(bankname,item); - // case 2: return hfi.getShorts(bankname,item); - // default: throw pybind11::type_error("HipoFileIterator: Invalid type int for entry: "+bankname+" "+item+"\n"); - // } - // } - // ); - //ADDED END - - //----------------------------------------------------------------------// - // Bind HIPO utils - py::class_ utils(m, "Utils"); - utils.def(py::init<>()); - - utils.def("tokenize", &hipo::utils::tokenize); - utils.def("substring", &hipo::utils::substring); - utils.def("findposition", &hipo::utils::findposition); - utils.def("ltrim", &hipo::utils::ltrim); - utils.def("rtrim", &hipo::utils::rtrim); - utils.def("trim", &hipo::utils::trim); - - utils.def("printLogo", &hipo::utils::printLogo); - - utils.def("getHeader", &hipo::utils::getHeader); - utils.def("getFileHeader", &hipo::utils::getFileHeader); - utils.def("getFileTrailer", &hipo::utils::getFileTrailer); - utils.def("getSConstruct", &hipo::utils::getSConstruct); - - utils.def("writeInt", &hipo::utils::writeInt); - utils.def("writeLong", &hipo::utils::writeLong); - utils.def("writeByte", &hipo::utils::writeByte); - - //----------------------------------------------------------------------// - // Bind HIPO benchmark - py::class_ benchmark(m, "Benchmark"); - benchmark.def(py::init<>()); - benchmark.def(py::init([](const char *name) { return new hipo::benchmark(name); })); - benchmark.def(py::init([](int freq) { return new hipo::benchmark(freq); })); - - benchmark.def("setName", &hipo::benchmark::setName); - benchmark.def("resume", &hipo::benchmark::resume); - benchmark.def("pause", &hipo::benchmark::pause); - benchmark.def("getTime", &hipo::benchmark::getTime); - benchmark.def("getTimeSec", &hipo::benchmark::getTimeSec); - benchmark.def("getCounter", &hipo::benchmark::getCounter); - benchmark.def("show", &hipo::benchmark::show); - -// class benchmark { -// private: - -// std::chrono::high_resolution_clock clock; -// std::chrono::time_point first, second; -// std::string benchmarkName; - -// long running_time; -// int counter; -// int printoutFrequency; - - // //----------------------------------------------------------------------// //NOTE: Ran into flat namespace error again when loading this in python after compilation - // // Bind HIPO datastream - // py::class_> datastream(m, "Datastream"); - // datastream.def(py::init<>()); - - // datastream.def("size", &hipo::datastream::size); //TODO: These are virtual methods... need to override above... - // datastream.def("position", static_cast(&hipo::datastream::position), "position"); - // datastream.def("position", static_cast(&hipo::datastream::position), "position"); - // datastream.def("open", &hipo::datastream::open); - // datastream.def("read", &hipo::datastream::read); - -// class datastream { -// private: -// std::ifstream inputStream; -// std::string remoteAddress; -// int streamType = 1; - - // //----------------------------------------------------------------------// //NOTE: Ran into flat namespace error again when loading this in python after compilation - // // Bind HIPO datastreamLocalFile - // py::class_ datastreamLocalFile(m, "DatastreamLocalFile"); - // datastreamLocalFile.def(py::init<>()); - - // datastreamLocalFile.def("size", &hipo::datastreamLocalFile::size); //TODO: These are virtual methods... need to override above... - // datastreamLocalFile.def("position", static_cast(&hipo::datastreamLocalFile::position), "position"); - // datastreamLocalFile.def("position", static_cast(&hipo::datastreamLocalFile::position), "position"); - // datastreamLocalFile.def("open", &hipo::datastreamLocalFile::open); - // datastreamLocalFile.def("read", &hipo::datastreamLocalFile::read); - -// class datastreamLocalFile { - -// private: -// std::ifstream inputStream; - - // //----------------------------------------------------------------------// //NOTE: Ran into flat namespace error again when loading this in python after compilation - // // Bind HIPO datastreamXrootd - // py::class_ datastreamXrootd(m, "DatastreamXrootd"); - // datastreamXrootd.def(py::init<>()); - - // datastreamXrootd.def("size", &hipo::datastreamXrootd::size); //TODO: These are virtual methods... need to override above... - // datastreamXrootd.def("position", static_cast(&hipo::datastreamXrootd::position), "position"); - // datastreamXrootd.def("position", static_cast(&hipo::datastreamXrootd::position), "position"); - // datastreamXrootd.def("open", &hipo::datastreamXrootd::open); - // datastreamXrootd.def("read", &hipo::datastreamXrootd::read); - -// class datastreamXrootd { -// private: -// #ifdef __XrootD__ -// kXR_unt16 open_mode = (kXR_ur); -// kXR_unt16 open_opts = (1); -// XrdClient *cli = NULL; -// #endif -// long streamPosition = 0; - - //----------------------------------------------------------------------// - // Bind HIPO schema - py::class_> schema(m, "Schema"); - schema.def(py::init<>()); - schema.def(py::init([](const char *name, int __groupid, int __itemid) { return new hipo::schema(name, __groupid, __itemid); })); - schema.def(py::init([](const hipo::schema &s) { return new hipo::schema(s); })); - - schema.def("parse", &hipo::schema::parse); - schema.def("getName", &hipo::schema::getName); - schema.def("getGroup", &hipo::schema::getGroup); - schema.def("getItem", &hipo::schema::getItem); - schema.def("getSizeForRows", &hipo::schema::getSizeForRows); - schema.def("getRowLength", &hipo::schema::getRowLength); - schema.def("getEntryOrder", &hipo::schema::getEntryOrder); - schema.def("exists", &hipo::schema::exists); - schema.def("getOffset", py::detail::overload_cast_impl()(&hipo::schema::getOffset, py::const_), "getOffset"); //NOTE: Need this syntax for const functions! - schema.def("getOffset", py::detail::overload_cast_impl()(&hipo::schema::getOffset, py::const_), "getOffset"); //NOTE: Need this syntax for const functions! - schema.def("getEntryType", &hipo::schema::getEntryType); - schema.def("getEntryName", &hipo::schema::getEntryName); - schema.def("getSchemaString", &hipo::schema::getSchemaString); - schema.def("getSchemaStringJson", &hipo::schema::getSchemaStringJson); - - schema.def_property_readonly("schemaEntriesMap", nullptr); - schema.def_property_readonly("schemaEntries", nullptr); - schema.def_property_readonly("groupid", &hipo::schema::getGroup); - schema.def_property_readonly("itemid", &hipo::schema::getItem); - schema.def_property_readonly("rowLength", &hipo::schema::getRowLength); - schema.def_property_readonly("warningCount", nullptr); - schema.def_property_readonly("schemaName", &hipo::schema::getName); - - schema.def("__repr__", //TODO: Test this function in python - [](hipo::schema &s) { std::string r("Schema : name = "+s.getName()+" , schemaString = "+s.getSchemaString()+"\n"); return r; } - ); - schema.def("__eq__", //TODO: Test this function in python - [](hipo::schema *s1, hipo::schema *s2) { return s1 == s2; } - ); - schema.def("__len__", //TODO: Test this function in python - [](hipo::schema &s) { return s.getRowLength(); } - ); - - //----------------------------------------------------------------------// - // Bind HIPO dictionary - py::class_> dictionary(m, "Dictionary"); - dictionary.def(py::init<>()); - - dictionary.def("getSchemaList", &hipo::dictionary::getSchemaList); - dictionary.def("addSchema", &hipo::dictionary::addSchema); - dictionary.def("hasSchema", &hipo::dictionary::hasSchema); - dictionary.def("getSchema", &hipo::dictionary::getSchema); - dictionary.def("parse", &hipo::dictionary::parse); - dictionary.def("show", &hipo::dictionary::show); - - // schema.def_property_readonly("factory", nullptr); - - dictionary.def("__repr__", //TODO: Test this function in python - [](hipo::dictionary &d) { - std::vector schemaList = d.getSchemaList(); - std::string r("Dictionary :\n"); - for(int idx = 0; idx> structure(m, "Structure"); - structure.def(py::init<>()); - structure.def(py::init([](int size) { return new hipo::structure(size); })); - structure.def(py::init([](int __group, int __item, std::string &str) { return new hipo::structure(__group, __item, str); })); - - structure.def("allocate", &hipo::structure::allocate); - structure.def("getSize", &hipo::structure::getSize); - structure.def("getType", &hipo::structure::getType); - structure.def("getGroup", &hipo::structure::getGroup); - structure.def("getItem", &hipo::structure::getItem); - structure.def("init", &hipo::structure::init); - structure.def("initNoCopy", &hipo::structure::initNoCopy); - structure.def("getAddress", &hipo::structure::getAddress); - structure.def("show", &hipo::structure::show); - structure.def("setSize", &hipo::structure::setSize); - structure.def("getIntAt", &hipo::structure::getIntAt); - structure.def("getShortAt", &hipo::structure::getShortAt); - structure.def("getByteAt", &hipo::structure::getByteAt); - structure.def("getFloatAt", &hipo::structure::getFloatAt); - structure.def("getDoubleAt", &hipo::structure::getDoubleAt); - structure.def("getLongAt", &hipo::structure::getLongAt); - structure.def("getStringAt", &hipo::structure::getStringAt); - structure.def("putIntAt", &hipo::structure::putIntAt); - structure.def("putShortAt", &hipo::structure::putShortAt); - structure.def("putByteAt", &hipo::structure::putByteAt); - structure.def("putFloatAt", &hipo::structure::putFloatAt); - structure.def("putDoubleAt", &hipo::structure::putDoubleAt); - structure.def("putLongAt", &hipo::structure::putLongAt); - structure.def("putStringAt", &hipo::structure::putStringAt); - structure.def("notify", &hipo::structure::notify); - - structure.def("__repr__", //TODO: Test this function in python - [](hipo::structure &s) { - std::string r("Structure : group = "); r += std::to_string(s.getGroup())+" , item = "+std::to_string(s.getItem())+" , type = "+std::to_string(s.getType())+" , length = "+std::to_string(s.getSize())+"\n"; - return r; - } - ); - structure.def("__eq__", //TODO: Test this function in python - [](hipo::structure *s1, hipo::structure *s2) { return s1 == s2; } - ); - structure.def("__len__", //TODO: Test this function in python - [](hipo::structure &s) { return s.getSize(); } - ); - - //----------------------------------------------------------------------// - // Bind HIPO bank - py::class_> bank(m, "Bank"); - bank.def(py::init<>()); - bank.def(py::init([](const hipo::schema& __schema) { return new hipo::bank(__schema); })); - bank.def(py::init([](const hipo::schema& __schema, int __rows) { return new hipo::bank(__schema, __rows); })); - - bank.def("getSchema", &hipo::bank::getSchema); - bank.def("getRows", &hipo::bank::getRows); - bank.def("setRows", &hipo::bank::setRows); - - bank.def("getInt", py::detail::overload_cast_impl()(&hipo::bank::getInt, py::const_), "getInt"); - bank.def("getInt", py::detail::overload_cast_impl()(&hipo::bank::getInt, py::const_), "getInt"); - bank.def("getShort", py::detail::overload_cast_impl()(&hipo::bank::getShort, py::const_), "getShort"); - bank.def("getShort", py::detail::overload_cast_impl()(&hipo::bank::getShort, py::const_), "getShort"); - bank.def("getByte", py::detail::overload_cast_impl()(&hipo::bank::getByte, py::const_), "getByte"); - bank.def("getByte", py::detail::overload_cast_impl()(&hipo::bank::getByte, py::const_), "getByte"); - bank.def("getFloat", py::detail::overload_cast_impl()(&hipo::bank::getFloat, py::const_), "getFloat"); - bank.def("getFloat", py::detail::overload_cast_impl()(&hipo::bank::getFloat, py::const_), "getFloat"); - bank.def("getDouble", py::detail::overload_cast_impl()(&hipo::bank::getDouble, py::const_), "getDouble"); - bank.def("getDouble", py::detail::overload_cast_impl()(&hipo::bank::getDouble, py::const_), "getDouble"); - bank.def("getLong", py::detail::overload_cast_impl()(&hipo::bank::getLong, py::const_), "getLong"); - bank.def("getLong", py::detail::overload_cast_impl()(&hipo::bank::getLong, py::const_), "getLong"); - - bank.def("getInts", py::detail::overload_cast_impl()(&hipo::bank::getInts, py::const_), "getInts"); - bank.def("getShorts", py::detail::overload_cast_impl()(&hipo::bank::getShorts, py::const_), "getShorts"); - bank.def("getBytes", py::detail::overload_cast_impl()(&hipo::bank::getBytes, py::const_), "getBytes"); - bank.def("getFloats", py::detail::overload_cast_impl()(&hipo::bank::getFloats, py::const_), "getFloats"); - bank.def("getDoubles", py::detail::overload_cast_impl()(&hipo::bank::getDoubles, py::const_), "getDoubles"); - bank.def("getLongs", py::detail::overload_cast_impl()(&hipo::bank::getLongs, py::const_), "getLongs"); - - bank.def("getDoubles", py::detail::overload_cast_impl()(&hipo::bank::getDoubles, py::const_), "getDoubles"); - - bank.def("putInt", py::detail::overload_cast_impl()(&hipo::bank::putInt), "putInt"); - bank.def("putInt", py::detail::overload_cast_impl()(&hipo::bank::putInt), "putInt"); - bank.def("putShort", py::detail::overload_cast_impl()(&hipo::bank::putShort), "putShort"); - bank.def("putShort", py::detail::overload_cast_impl()(&hipo::bank::putShort), "putShort"); - bank.def("putByte", py::detail::overload_cast_impl()(&hipo::bank::putByte), "putByte"); - bank.def("putByte", py::detail::overload_cast_impl()(&hipo::bank::putByte), "putByte"); - bank.def("putFloat", py::detail::overload_cast_impl()(&hipo::bank::putFloat), "putFloat"); - bank.def("putFloat", py::detail::overload_cast_impl()(&hipo::bank::putFloat), "putFloat"); - bank.def("putDouble", py::detail::overload_cast_impl()(&hipo::bank::putDouble), "putDouble"); - bank.def("putDouble", py::detail::overload_cast_impl()(&hipo::bank::putDouble), "putDouble"); - bank.def("putLong", py::detail::overload_cast_impl()(&hipo::bank::putLong), "putLong"); - bank.def("putLong", py::detail::overload_cast_impl()(&hipo::bank::putLong), "putLong"); - - bank.def("putInts", py::detail::overload_cast_impl>()(&hipo::bank::putInts), "putInts"); - bank.def("putShorts", py::detail::overload_cast_impl>()(&hipo::bank::putShorts), "putShorts"); - bank.def("putBytes", py::detail::overload_cast_impl>()(&hipo::bank::putBytes), "putBytes"); - bank.def("putFloats", py::detail::overload_cast_impl>()(&hipo::bank::putFloats), "putFloats"); - bank.def("putDoubles", py::detail::overload_cast_impl>()(&hipo::bank::putDoubles), "putDoubles"); - bank.def("putLongs", py::detail::overload_cast_impl>()(&hipo::bank::putLongs), "putLongs"); - - bank.def("show", &hipo::bank::show); - bank.def("reset", &hipo::bank::reset); - bank.def("notify", &hipo::bank::notify); - - bank.def_property_readonly("bankSchema", &hipo::bank::getSchema); //NOTE: Not really necessary. - bank.def_property_readonly("bankRows", &hipo::bank::getRows); - - bank.def("__repr__", //TODO: Test this function in python - [](hipo::bank &b) { - std::string r("Bank : name = "); r += b.getSchema().getName()+" , rows = "+std::to_string(b.getRows())+"\n"; - for (int i = 0; i < b.getSchema().getEntries(); i++) { - std::string x("\t"); x += b.getSchema().getEntryName(i)+" : "; r += x; - for (int k = 0; k < b.getRows(); k++) { - if (b.getSchema().getEntryType(i) < 4) { - r += std::to_string(b.getInt(i,k))+" "; // std::string y("%8d ", b.getInt(i,k)); r += y; // %8d - } else { - if (b.getSchema().getEntryType(i)==4) { - r += std::to_string(b.getFloat(i,k))+" "; // std::string y("%8.5f ",b.getFloat(i,k)); r += y; // %8.5f - } - if (b.getSchema().getEntryType(i)==5) { - r += std::to_string(b.getDouble(i,k))+" "; // std::string y("%8.5f ",b.getDouble(i,k)); r += y; // %8.5f - } - if (b.getSchema().getEntryType(i)==8) { - r += std::to_string(b.getLong(i,k))+" "; // std::string y("%14ld ", b.getLong(i,k)); r += y; // %14ld - } - } // else - } // for(int k = 0; k < b.getRows(); k++) - r += "\n"; - } // for(int i = 0; i < b.getSchema().getEntries(); i++) - return r; - } // [](hipo::bank &b) - ); - bank.def("__eq__", //TODO: Test this function in python - [](hipo::bank *b1, hipo::bank *b2) { return b1 == b2; } - ); - bank.def("__len__", //TODO: Test this function in python - [](hipo::bank &b) { return b.getRows(); } - ); - - //----------------------------------------------------------------------// - // Bind HIPO event - py::class_> event(m, "Event"); - event.def(py::init<>()); - event.def(py::init([](int size) { return new hipo::event(size); })); - - event.def("show", &hipo::event::show); - event.def("init", static_cast&)>(&hipo::event::init), "init"); - event.def("init", static_cast(&hipo::event::init), "init"); - event.def("getStructure", static_cast(&hipo::event::getStructure), "getStructure"); - event.def("getStructure", static_cast(&hipo::event::getStructure), "getStructure"); - // event.def_static("getStructure", static_cast(&hipo::event::getStructure), "getStructure"); //TODO: //NOTE: Not sure why this doesn't work right now... - event.def("getTag", &hipo::event::getTag); - event.def("read", &hipo::event::read); - event.def("addStructure", &hipo::event::addStructure); - event.def("getStructurePosition", static_cast (hipo::event::*)(int, int)>(&hipo::event::getStructurePosition), "getStructurePosition"); - // event.def_static("getStructurePosition", static_cast (hipo::event::*)(const char*, int, int)>(&hipo::event::getStructurePosition), "getStructurePosition"); //TODO: //NOTE: Not sure why this doesn't work right now... same as above is a static method maybe that is why... - event.def("getEventBuffer", &hipo::event::getEventBuffer); - event.def("getSize", &hipo::event::getSize); - event.def("reset", &hipo::event::reset); - event.def("getStructureNoCopy", &hipo::event::getStructureNoCopy); - - // event.def_property_readonly("databuffer", nullptr); //NOTE: Not really necessary. - - event.def("__repr__", //TODO: Test this function in python - [](hipo::event &e) { - std::string r("Event : size = "); r += std::to_string(e.getSize())+"\n"; - int position = 16; - int eventSize = *(reinterpret_cast(&e.getEventBuffer()[4])); - while(position+8(&e.getEventBuffer()[position])); - uint8_t iid = *(reinterpret_cast(&e.getEventBuffer()[position+2])); - uint8_t type = *(reinterpret_cast(&e.getEventBuffer()[position+3])); - int length = *(reinterpret_cast(&e.getEventBuffer()[position+4])); - r += "\tgroup = "+std::to_string(gid)+" , item = "+std::to_string(iid)+" , type = "+std::to_string(type)+" , length = "+std::to_string(length)+"\n"; - position += (length + 8); - } - return r; - } // [](hipo::event &e) - ); - event.def("__eq__", //TODO: Test this function in python - [](hipo::event *e1, hipo::event *e2) { return e1 == e2; } - ); - event.def("__len__", //TODO: Test this function in python - [](hipo::event &e) { return e.getSize(); } - ); - - //----------------------------------------------------------------------// - // Bind HIPO readerIndex - py::class_ readerIndex(m, "ReaderIndex"); //NOTE: Can't use trampoline class because you get this error: "Cannot use an alias class with a non-polymorphic type" - readerIndex.def(py::init<>([]() { return new hipo::readerIndex(); })); - - readerIndex.def("canAdvance", &hipo::readerIndex::canAdvance); - readerIndex.def("advance", &hipo::readerIndex::advance); - - readerIndex.def("canAdvanceInRecord", &hipo::readerIndex::canAdvanceInRecord); - readerIndex.def("loadRecord", &hipo::readerIndex::loadRecord); - readerIndex.def("gotoEvent", &hipo::readerIndex::gotoEvent); - readerIndex.def("gotoRecord", &hipo::readerIndex::gotoRecord); - - readerIndex.def("getEventNumber", &hipo::readerIndex::getEventNumber); - readerIndex.def("getRecordNumber", &hipo::readerIndex::getRecordNumber); - readerIndex.def("getRecordEventNumber", &hipo::readerIndex::getRecordEventNumber); - readerIndex.def("getMaxEvents", &hipo::readerIndex::getMaxEvents); - readerIndex.def("addSize", &hipo::readerIndex::addSize); - readerIndex.def("addPosition", &hipo::readerIndex::addPosition); - readerIndex.def("getPosition", &hipo::readerIndex::getPosition); - - readerIndex.def("getNRecords", &hipo::readerIndex::getNRecords); - readerIndex.def("rewind", &hipo::readerIndex::rewind); - readerIndex.def("clear", &hipo::readerIndex::clear); - readerIndex.def("reset", &hipo::readerIndex::reset); - - // readerIndex.def_property_readonly("recordEvents", nullptr); //NOTE: Not really necessary. - // readerIndex.def_property_readonly("recordPosition", nullptr); - readerIndex.def_property_readonly("currentRecord", &hipo::readerIndex::getRecordNumber); - readerIndex.def_property_readonly("currentEvent", &hipo::readerIndex::getEventNumber); - readerIndex.def_property_readonly("currentRecordEvent", &hipo::readerIndex::getRecordEventNumber); - - readerIndex.def("__repr__", - [](hipo::readerIndex &r){ - std::string x("Reader Index : "); - x += "nrecords = "+std::to_string(r.getNRecords())+" , record number = "+std::to_string(r.getRecordNumber())+" , event number = "+std::to_string(r.getEventNumber())+"\n"; - return x; - } - ); - - readerIndex.def("__eq__", //TODO: Test this function in python - [](hipo::readerIndex *r1, hipo::readerIndex *r2) { return r1 == r2; } - ); - readerIndex.def("__len__", //TODO: Test this function in python - [](hipo::readerIndex &r) { return r.getNRecords(); } - ); - - //----------------------------------------------------------------------// - // Bind HIPO reader - py::class_ reader(m, "Reader"); - reader.def(py::init<>([]() { return new hipo::reader(); })); - reader.def(py::init([](const hipo::reader &r) { return new hipo::reader(r); })); - - reader.def("about", &hipo::reader::about); - reader.def("readDictionary", &hipo::reader::readDictionary); - reader.def("getStructure", &hipo::reader::getStructure); - reader.def("getStructureNoCopy", &hipo::reader::getStructureNoCopy); - reader.def("open", &hipo::reader::open); - reader.def("setTags", static_cast(&hipo::reader::setTags), "setTags"); - reader.def("setTags", static_cast)>(&hipo::reader::setTags), "setTags"); - reader.def("setVerbose", &hipo::reader::setVerbose); - - reader.def("hasNext", &hipo::reader::hasNext); - reader.def("next", static_cast(&hipo::reader::next), "next"); - reader.def("next", static_cast(&hipo::reader::next), "next"); - reader.def("gotoEvent", &hipo::reader::gotoEvent); - reader.def("gotoRecord", &hipo::reader::gotoRecord); - reader.def("read", &hipo::reader::read); - reader.def("printWarning", &hipo::reader::printWarning); - - reader.def("getNRecords", &hipo::reader::getNRecords); - reader.def("nextInRecord", &hipo::reader::nextInRecord); - reader.def("loadRecord", &hipo::reader::loadRecord); - reader.def("getEntries", &hipo::reader::getEntries); - - reader.def("__repr__", - [](hipo::reader &r){ - std::string x("Reader : "); - x += "nrecords = "+std::to_string(r.getNRecords())+" , entries = "+std::to_string(r.getEntries())+"\n"; - return x; - } - ); - reader.def("__eq__", //TODO: Test this function in python - [](hipo::reader *r1, hipo::reader *r2) { return r1 == r2; } - ); - reader.def("__len__", //TODO: Test this function in python - [](hipo::reader &r) { return r.getNRecords(); } - ); - - //----------------------------------------------------------------------// - // Bind HIPO data - py::class_ data(m, "Data"); - data.def(py::init<>()); - - data.def("setDataPtr", &hipo::data::setDataPtr); - data.def("setDataSize", &hipo::data::setDataSize); - data.def("setDataOffset", &hipo::data::setDataOffset); - data.def("setDataEndianness", &hipo::data::setDataEndianness); - - data.def("getEvioPtr", &hipo::data::getEvioPtr); - data.def("getEvioSize", &hipo::data::getEvioSize); - data.def("getDataPtr", &hipo::data::getDataPtr); - data.def("getDataSize", &hipo::data::getDataSize); - data.def("getDataOffset", &hipo::data::getDataOffset); - data.def("getDataEndianness", &hipo::data::getDataEndianness); - - data.def_property_readonly("data_ptr", &hipo::data::getDataPtr); - data.def_property_readonly("data_size", &hipo::data::getDataSize); - data.def_property_readonly("data_endianness", &hipo::data::getDataEndianness); - data.def_property_readonly("data_offset", &hipo::data::getDataOffset); - - data.def("__repr__", - [](hipo::data &d){ - std::string r("Data : "); - r += "size = "+std::to_string(d.getDataSize())+"\n"; - return r; - } - ); - data.def("__eq__", //TODO: Test this function in python - [](hipo::data *d1, hipo::data *d2) { return d1 == d2; } - ); - data.def("__len__", //TODO: Test this function in python - [](hipo::data &d) { return d.getDataSize(); } - ); - - //----------------------------------------------------------------------// - // Bind HIPO record - py::class_ record(m, "Record"); - record.def(py::init<>()); - - record.def("readRecord", static_cast(&hipo::record::readRecord), "readRecord"); - record.def("readRecord", static_cast(&hipo::record::readRecord), "readRecord"); - record.def("readRecord__", &hipo::record::readRecord__); - record.def("getEventCount", &hipo::record::getEventCount); - record.def("getRecordSizeCompressed", &hipo::record::getRecordSizeCompressed); - - record.def("readEvent", &hipo::record::readEvent); - record.def("readHipoEvent", &hipo::record::readHipoEvent); - record.def("getData", &hipo::record::getData); - - record.def("getReadBenchmark", &hipo::record::getReadBenchmark); - record.def("getUnzipBenchmark", &hipo::record::getUnzipBenchmark); - record.def("getIndexBenchmark", &hipo::record::getIndexBenchmark); - - record.def_property_readonly("readBenchmark", &hipo::record::getReadBenchmark); - record.def_property_readonly("unzipBenchmark", &hipo::record::getUnzipBenchmark); - record.def_property_readonly("indexBenchmark", &hipo::record::getIndexBenchmark); - - record.def("__repr__", - [](hipo::record &r){ - std::string x("Record : "); - x += "event count = "+std::to_string(r.getEventCount())+"\n"; - return x; - } - ); - record.def("__eq__", //TODO: Test this function in python - [](hipo::record *r1, hipo::record *r2) { return r1 == r2; } - ); - record.def("__len__", //TODO: Test this function in python - [](hipo::record &r) { return r.getEventCount(); } - ); - - //----------------------------------------------------------------------// - // Bind HIPO recordbuilder - py::class_> recordbuilder(m, "RecordBuilder"); - recordbuilder.def(py::init<>([]() { return new hipo::recordbuilder(); })); - recordbuilder.def(py::init<>([](int maxEvents, int maxLength) { return new hipo::recordbuilder(maxEvents,maxLength); })); - - recordbuilder.def("addEvent", static_cast(&hipo::recordbuilder::addEvent), "addEvent"); - recordbuilder.def("addEvent", static_cast&, int, int)>(&hipo::recordbuilder::addEvent), "addEvent"); - - recordbuilder.def("getUserWordOne", &hipo::recordbuilder::getUserWordOne); - recordbuilder.def("getUserWordTwo", &hipo::recordbuilder::getUserWordTwo); - recordbuilder.def("setUserWordOne", &hipo::recordbuilder::setUserWordOne); - recordbuilder.def("setUserWordTwo", &hipo::recordbuilder::setUserWordTwo); - - recordbuilder.def("getRecordSize", &hipo::recordbuilder::getRecordSize); - recordbuilder.def("getEntries", &hipo::recordbuilder::getEntries); - recordbuilder.def("getRecordBuffer", &hipo::recordbuilder::getRecordBuffer); - recordbuilder.def("reset", &hipo::recordbuilder::reset); - recordbuilder.def("build", &hipo::recordbuilder::build); - - recordbuilder.def_property_readonly("bufferUserWordOne", &hipo::recordbuilder::getUserWordOne); - recordbuilder.def_property_readonly("bufferUserWordTwo", &hipo::recordbuilder::getUserWordTwo); - - recordbuilder.def("__repr__", - [](hipo::recordbuilder &r){ - std::string x("Recordbuilder : "); - x += "entries = "+std::to_string(r.getEntries())+" , userWordOne = "+std::to_string(r.getUserWordOne())+" , userWordTwo = "+std::to_string(r.getUserWordTwo())+"\n"; - return x; - } - ); - recordbuilder.def("__eq__", //TODO: Test this function in python - [](hipo::recordbuilder *r1, hipo::recordbuilder *r2) { return r1 == r2; } - ); - recordbuilder.def("__len__", //TODO: Test this function in python - [](hipo::recordbuilder &r) { return r.getEntries(); } - ); - - //----------------------------------------------------------------------// - // Bind HIPO writer - py::class_> writer(m, "Writer"); - writer.def(py::init<>([]() { return new hipo::writer(); })); - - writer.def("addEvent", static_cast(&hipo::writer::addEvent), "addEvent"); - writer.def("addEvent", static_cast&, int)>(&hipo::writer::addEvent), "addEvent"); - writer.def("writeRecord", &hipo::writer::writeRecord); - writer.def("open", &hipo::writer::open); - writer.def("close", &hipo::writer::close); - writer.def("showSummary", &hipo::writer::showSummary); - writer.def("addDictionary", &hipo::writer::addDictionary); - writer.def("getDictionary", &hipo::writer::getDictionary); - writer.def("setUserIntegerOne", &hipo::writer::setUserIntegerOne); - writer.def("setUserIntegerTwo", &hipo::writer::setUserIntegerTwo); - writer.def("flush", &hipo::writer::flush); - - writer.def_property_readonly("writerDictionary", &hipo::writer::getDictionary); - - writer.def("__repr__", - [](hipo::writer &w){ - std::string r("Writer : "); - return r+"\n"; - } - ); - writer.def("__eq__", //TODO: Test this function in python - [](hipo::writer *w1, hipo::writer *w2) { return w1 == w2; } - ); - - //----------------------------------------------------------------------// - // Documentation - - // m.doc() = R"pbdoc( - // HipopyBind plugin - // ----------------- - - // .. currentmodule:: hipopybind - - // .. autosummary:: - // :toctree: _generate - - // add - // subtract - // )pbdoc"; - - // m.def("add", &add, R"pbdoc( - // Add two numbers - - // Some other explanation about the add function. - // )pbdoc"); - - // m.def("subtract", [](int i, int j) { return i - j; }, R"pbdoc( - // Subtract two numbers - - // Some other explanation about the subtract function. - // )pbdoc"); - -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif -} //PYBIND11_MODULE diff --git a/tests/test_basic.py b/tests/test_basic.py index b5f1a69..fbbd680 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -1,4 +1,4 @@ import hipopybind as m def test_main(): - assert m.__version__ == "1.1.3" + assert m.__version__ == "2.0.1" diff --git a/tutorials/write.py b/tutorials/write.py index ba7a232..9952f41 100644 --- a/tutorials/write.py +++ b/tutorials/write.py @@ -60,7 +60,7 @@ b.setRows(nrows) for name in names: values = np.random.random(size=(nrows,)) - b.putDoubles(name,values) + hb.putDoubles(b,name,values) # Show bank data print("Iteration",i)