diff --git a/.github/workflows/c_tests.yml b/.github/workflows/c_tests.yml new file mode 100644 index 000000000..0aa39e785 --- /dev/null +++ b/.github/workflows/c_tests.yml @@ -0,0 +1,24 @@ +name: "C/C++ Tests" +on: + pull_request: + branches: [ main ] + +jobs: + cpp_test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y cmake build-essential libboost-filesystem-dev libboost-thread-dev libboost-chrono-dev liblz4-dev doxygen + + - name: Configure CMake + run: cmake -S . -B build -DMAKE_EXAMPLES=1 + + - name: Build + run: cmake --build build -- -j$(nproc) + + - name: Run tests + working-directory: build + run: ctest --output-on-failure diff --git a/.github/workflows/java_tests.yml b/.github/workflows/java_tests.yml new file mode 100644 index 000000000..58e3fd3f0 --- /dev/null +++ b/.github/workflows/java_tests.yml @@ -0,0 +1,30 @@ +name: Java Tests (EVIO-6) + +on: + pull_request: + branches: [ main ] + +jobs: + java_test: + name: Run JUnit Tests + runs-on: ubuntu-latest + container: + image: maven:3.9-eclipse-temurin-17-alpine + + steps: + - name: Check out code + uses: actions/checkout@v3 + + # Uncomment to set up Java and Maven manually, if desired + # # (and probably comment out the container image above) + # - name: Set up JDK 17 + # uses: actions/setup-java@v3 + # with: + # distribution: temurin + # java-version: 17 + # Install maven by hand (if not in container image) + # - name: Maven Setup + # run: sudo apt-get update && apt-get install -y maven + + - name: Build and run tests + run: mvn --batch-mode test diff --git a/.gitignore b/.gitignore index 2e3796a9e..8457866f8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,21 +2,38 @@ .* # ignore all files/dirs beginning with _ _* +# EventViewer Jar +JEventViewer*.jar # except these two !.gitignore !.github/ +# ignore any .evio, .ev, or .hipo extension files (+ other common file types) +*.evio* +*.ev +*.hipo +*.dat +*.txt +*.out.* +perf.data* + +# test programs to ignore +src/main/java/org/jlab/coda/jevio/dev/* + # ignore generated files/dirs cmake-* cmake_install.cmake coda.pyc build/ jbuild/ +lib/ +bin/ doxyerrors.log -/target/ +target/ doc/javadoc doc/doxygen/CC doc/doxygen/C doc/source/ config.log + diff --git a/CMakeLists.txt b/CMakeLists.txt index ecc01d68e..84f4ba016 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,54 +1,25 @@ -# To use this, do the following the first time: -# % mkdir build -# % cd build -# % cmake .. -DCMAKE_BUILD_TYPE=release -DMAKE_EXAMPLES=1 -# % make (install) -# -# To only build C libraries and executables, replace the 3rd command above with -# % cmake .. -DCMAKE_BUILD_TYPE=release -DC_ONLY=1 -# -# After doing this once can do the following to recompile -# % cd -# % cmake --build build (--target install) -# -# The installation will default to the CODA env variable if it's defined -# This can be overridden on command line (in build dir) with: -# % cmake -DCODA_INSTALL= .. -# % make install - -# This call must be placed BEFORE "project" command cmake_minimum_required(VERSION 3.22) - -# in cmake v 3.30 and later, new way to look for boost -if(POLICY CMP0167) - cmake_policy(SET CMP0167 NEW) -endif() - -project(evio VERSION 6.0) - -# This is better to be on the top of CMake! -include(GNUInstallDirs) - -# Allow users to set C++ standard via -DCMAKE_CXX_STANDARD -# Only default to C++11 if not already set -if(NOT CMAKE_CXX_STANDARD) - set(CMAKE_CXX_STANDARD 11) -endif() -set(CMAKE_CXX_STANDARD_REQUIRED True) +project(evio VERSION 6.0 LANGUAGES C CXX) + +# Build options (and defaults) +option(C_ONLY "SKIP building C++ library, build C only" OFF) +option(MAKE_EXAMPLES "Build example/test programs" OFF) +option(USE_FILESYSTEMLIB "Use C++17 instead of Boost" OFF) +option(DISRUPTOR_FETCH "Allow CMake to download Disruptor if not found" ON) +# C/C++ standard and build options +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_DEBUG_POSTFIX -dbg) add_compile_options(-Wall) +include(GNUInstallDirs) +include(FindPackageHandleStandardArgs) # find_package_handle_standard_args() +include(CTest) + # Add custom find_package for Disruptor list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules") -# Build examples by default, but allow disabling -option(MAKE_EXAMPLES "Build examples and tests" ON) -option(C_ONLY "Build only C library (no C++)" OFF) - -# Option to enable/disable usage of -option(USE_FILESYSTEMLIB "Use C++17 instead of Boost" OFF) - -# Later, when creating your library or executable, add a compile definition if it's ON +# Add a compile definition if USE_FILESYSTEMLIB set, for when creating your library or executable if(USE_FILESYSTEMLIB) add_compile_definitions(USE_FILESYSTEMLIB=1) endif() @@ -57,7 +28,7 @@ endif() set(LIBRARY_OUTPUT_PATH ${CMAKE_BINARY_DIR}/lib) set(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/bin) -# Set architecture-related string for installation consistent with scons file +# Set architecture-related string for installation (consistent with scons) set(ARCH ${CMAKE_HOST_SYSTEM_NAME}-${CMAKE_HOST_SYSTEM_PROCESSOR}) message(STATUS "ARCH = " ${ARCH}) @@ -76,177 +47,13 @@ elseif (DEFINED ENV{CODA}) set(CMAKE_INSTALL_INCLUDEDIR $ENV{CODA}/common/include) message(STATUS "CMAKE_INSTALL_PREFIX reset to " ${CMAKE_INSTALL_PREFIX}) else() - # Always define installation, even without CODA, using the default CMAKE_INSTALL_PREFIX + # Use default CMAKE_INSTALL_PREFIX set(INSTALL_DIR_DEFINED 1) - message(STATUS "No CODA environment - will install to default location: ${CMAKE_INSTALL_PREFIX}") - # Don't set CMAKE_INSTALL_PREFIX here - use the CMake default + message(STATUS "Installing to default location: ${CMAKE_INSTALL_PREFIX}") endif() -set(C_HEADER_FILES - src/libsrc/evio.h - src/libsrc/msinttypes.h -) - -set(C_LIB_FILES - src/libsrc/evio.c - src/libsrc/evioswap.c - src/libsrc/eviofmt.c - src/libsrc/eviofmtdump.c - src/libsrc/eviofmtswap.c - src/libsrc/eviocopy.c -) - -set(CPP_HEADER_FILES - src/libsrc++/FileWritingSupport.h - src/libsrc++/FileEventIndex.h - src/libsrc++/EvioException.h - src/libsrc++/ByteOrder.h - src/libsrc++/ByteBuffer.h - src/libsrc++/HeaderType.h - src/libsrc++/Compressor.h - src/libsrc++/FileHeader.h - src/libsrc++/RecordHeader.h - src/libsrc++/RecordInput.h - src/libsrc++/RecordOutput.h - src/libsrc++/Writer.h - src/libsrc++/WriterMT.h - src/libsrc++/EvioNode.h - src/libsrc++/DataType.h - src/libsrc++/StructureType.h - src/libsrc++/RecordNode.h - src/libsrc++/Reader.h - src/libsrc++/RecordSupply.h - src/libsrc++/RecordRingItem.h - src/libsrc++/Util.h - src/libsrc++/EventWriter.h - src/libsrc++/EventWriterV4.h - src/libsrc++/RecordCompressor.h - src/libsrc++/BaseStructure.h - src/libsrc++/BaseStructureHeader.h - src/libsrc++/CompositeData.h - src/libsrc++/BankHeader.h - src/libsrc++/SegmentHeader.h - src/libsrc++/TagSegmentHeader.h - src/libsrc++/EvioBank.h - src/libsrc++/EvioSegment.h - src/libsrc++/EvioTagSegment.h - src/libsrc++/EvioEvent.h - src/libsrc++/EventParser.h - src/libsrc++/IEvioFilter.h - src/libsrc++/IEvioListener.h - src/libsrc++/EventParser.h - src/libsrc++/EventHeaderParser.h - src/libsrc++/StructureTransformer.h - src/libsrc++/IBlockHeader.h - src/libsrc++/IEvioReader.h - src/libsrc++/EvioReader.h - src/libsrc++/EvioReaderV4.h - src/libsrc++/BlockHeaderV2.h - src/libsrc++/BlockHeaderV4.h - src/libsrc++/EvioReaderV6.h - src/libsrc++/EvioCompactReader.h - src/libsrc++/IEvioCompactReader.h - src/libsrc++/EvioXMLDictionary.h - src/libsrc++/EvioDictionaryEntry.h - src/libsrc++/pugixml.hpp - src/libsrc++/pugiconfig.hpp - src/libsrc++/EvioCompactReaderV4.h - src/libsrc++/EvioCompactReaderV6.h - src/libsrc++/EvioCompactStructureHandler.h - src/libsrc++/EventBuilder.h - src/libsrc++/CompactEventBuilder.h - src/libsrc++/EvioSwap.h - src/libsrc++/StructureFinder.h - src/libsrc++/eviocc.h - src/test/TestBase.h -) - -set(CPP_LIB_FILES - src/libsrc++/FileEventIndex.cpp - src/libsrc++/ByteOrder.cpp - src/libsrc++/ByteBuffer.cpp - src/libsrc++/HeaderType.cpp - src/libsrc++/Compressor.cpp - src/libsrc++/FileHeader.cpp - src/libsrc++/RecordHeader.cpp - src/libsrc++/RecordInput.cpp - src/libsrc++/RecordOutput.cpp - src/libsrc++/Writer.cpp - src/libsrc++/WriterMT.cpp - src/libsrc++/EvioNode.cpp - src/libsrc++/DataType.cpp - src/libsrc++/StructureType.cpp - src/libsrc++/Reader.cpp - src/libsrc++/RecordSupply.cpp - src/libsrc++/RecordRingItem.cpp - src/libsrc++/EventWriter.cpp - src/libsrc++/EventWriterV4.cpp - src/libsrc++/BaseStructure.cpp - src/libsrc++/BaseStructureHeader.cpp - src/libsrc++/CompositeData.cpp - src/libsrc++/BankHeader.cpp - src/libsrc++/SegmentHeader.cpp - src/libsrc++/TagSegmentHeader.cpp - src/libsrc++/EventParser.cpp - src/libsrc++/EvioReader.cpp - src/libsrc++/EvioReaderV4.cpp - src/libsrc++/EvioReaderV6.cpp - src/libsrc++/EvioCompactReader.cpp - src/libsrc++/EvioXMLDictionary.cpp - src/libsrc++/pugixml.cpp - src/libsrc++/EvioCompactReaderV4.cpp - src/libsrc++/EvioCompactReaderV6.cpp - src/libsrc++/EvioCompactStructureHandler.cpp - src/libsrc++/EventBuilder.cpp - src/libsrc++/CompactEventBuilder.cpp - src/test/TestBase.cpp -) - -set(TEST - src/test/CompactBuilder_Test.cpp - src/test/CompactReaderAddSubtractTest.cpp - src/test/CompactReaderBugTest.cpp - src/test/CompositeTester.cpp - src/test/EvioCompStructHandlerTest.cpp - src/test/DictTest.cpp - src/test/Hipo_Test.cpp - src/test/IndexArrayLenTest.cpp - src/test/ReadWriteTest.cpp - src/test/ReadWriteV4Test.cpp - src/test/RecordSupplyTest.cpp - src/test/RingBufferTest.cpp - src/test/Tree_Buf_Composite_Builder_Test.cpp - src/test/hallDEventFiles.cpp - src/test/FileWriteTest.cpp - src/test/SwapTest.cpp - src/test/ReadSequenceTest.cpp - src/test/SequentialReaderTest.cpp - src/test/SplitTest2.cpp - ) - -set(TESTC - src/testC/evReadPipe.c - src/testC/evtest.c - src/testC/evtest2.c - src/testC/evtest3.c - src/testC/evtestAppend.c - src/testC/evtestBuf.c - src/testC/evtestBuf2.c - src/testC/evTestFile.c - src/testC/evtestWriteFile.c - src/testC/evtestRead.c - src/testC/evtestRio.c - src/testC/evtestSock.c - src/testC/evWritePipe.c - src/testC/splitTest.c - src/testC/splitNameTest.c -) - -include(FindPackageHandleStandardArgs) - -# search for boost libs +# Boost libs find_package(Boost REQUIRED COMPONENTS system thread chrono filesystem) - if (Boost_FOUND) message(STATUS "Boost Found: libs = " ${Boost_LIBRARIES} ", include dirs = " ${Boost_INCLUDE_DIRS}) link_directories(${Boost_LIBRARY_DIRS}) @@ -255,14 +62,11 @@ elseif(NOT C_ONLY) message(FATAL_ERROR "Boost cannot be found, cmake will exit." ) endif() -# -------------------------------------------------- -# LZ4 detection -# -------------------------------------------------- +# LZ4 library find_path(LZ4_INCLUDE_DIR NAMES lz4.h PATHS /usr/local/include /usr/include /usr/include/linux ) - find_library(LZ4_LIBRARY NAMES lz4 liblz4 PATHS @@ -273,16 +77,12 @@ find_library(LZ4_LIBRARY /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu/mesa ) - -include(FindPackageHandleStandardArgs) find_package_handle_standard_args(LZ4 REQUIRED_VARS LZ4_LIBRARY LZ4_INCLUDE_DIR HANDLE_COMPONENTS ) - if(NOT LZ4_FOUND) message(STATUS "LZ4 NOT found with the initial search; trying alternative approach ...") - # Fallback or manual search logic if you have other unique paths or want # to do something different. Or just fail here if that's your preference. # e.g. message(FATAL_ERROR "No LZ4 found!") @@ -290,8 +90,21 @@ else() message(STATUS "LZ4 Found: library = ${LZ4_LIBRARY}, include = ${LZ4_INCLUDE_DIR}") endif() +# C source files +file(GLOB C_HEADER_FILES "src/libsrc/*.h") +file(GLOB C_LIB_FILES "src/libsrc/*.c") +# C++ source files +file(GLOB CPP_LIB_FILES "src/libsrc++/*.cpp") +file(GLOB CPP_HEADER_FILES "src/libsrc++/*.h") +# A few extras required +if(MAKE_EXAMPLES) + file(GLOB TEST "src/test/cpp/*.cpp") + file(GLOB TESTC "src/test/c/*.c") + list(APPEND CPP_HEADER_FILES src/test/cpp/EvioTestHelper.h) + # list(APPEND CPP_LIB_FILES src/test/cpp/EvioTestHelper.h) +endif() -# If building C++ version, handle Disruptor dependency +# BUILD C++ LIBRARY (unless otherwise specified) if(NOT C_ONLY) find_package(Disruptor REQUIRED) # or find_package(Disruptor) @@ -439,19 +252,22 @@ if(NOT C_ONLY) ) endif() +# Unit testing setup +add_test(NAME EvioWriteAndReadBack_builder COMMAND bin/EvioWriteAndReadBack_builder 10) # Uninstall target -if(NOT TARGET uninstall) - configure_file( - "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in" - "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" - IMMEDIATE @ONLY - ) - - add_custom_target(uninstall - COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake - ) -endif() +# Removed for now, not yet compatible with building disruptor-cpp internally +# if(NOT TARGET uninstall) +# configure_file( +# "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in" +# "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" +# IMMEDIATE @ONLY +# ) + +# add_custom_target(uninstall +# COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake +# ) +# endif() # Options printout message(STATUS "Options table: ") diff --git a/LICENSE.txt b/LICENSE.txt index 532710ce4..2b502ba70 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2020, Jefferson Science Associates, LLC. All Rights Reserved. Redistribution +Copyright (c) 2025, Jefferson Science Associates, LLC. All Rights Reserved. Redistribution and use in source and binary forms, with or without modification, are permitted as a licensed user provided that the following conditions are met: diff --git a/README.md b/README.md index ac8120009..c7bdb78be 100644 --- a/README.md +++ b/README.md @@ -1,400 +1,89 @@ ----------------------------- - -# **EVIO 6.0 SOFTWARE PACKAGE** - ----------------------------- - -EVIO stands for EVent Input/Output and contains libraries which read & write -data in its own unique format. It was created by the Data Acquisition (DAQ) group -and is now maintained by the Experimental Physics Software and Computing Infrastructure -(EPSCI) group at Thomas Jefferson National Accelerator Facility (Jefferson Lab). -It was originally used in the online DAQ as part of the CODA software, -but is now being used offline and in other applications as well. - -There is a C library as well as an independent C++ library which -is a port from the Java version. There are a few utility programs included. -The Java version is a library with an extensive, user-friendly -interface. There is an additional package containing a GUI for looking at -evio format data at the byte level. - -If you only plan to run C/C++ applications you can skip the Java -installation. If you only plan to use Java applications you can -you can skip the C/C++ installation. - -### **Main evio links:** +# **EVIO 6 SOFTWARE PACKAGE** - [EVIO Home Page](https://coda.jlab.org/drupal/content/event-io-evio/) +EVIO stands for EVent Input/Output, a unique data format developed by Jefferson Lab. +It was created by the Data Acquisition (DAQ) group and is maintained by the +Experimental Physics Software and Computing Infrastructure (EPSCI) group at Thomas +Jefferson National Accelerator Facility (JLab). - [EVIO on GitHub (**evio-6.0** branch)](https://github.com/JeffersonLab/evio) +This software repository allows one to read & write evio format data, within +either a C/C++ or Java programming environment. -### **GUI for viewing data in evio format:** +# **Getting Started** - [EVIO Event Viewer Home Page](https://coda.jlab.org/drupal/content/graphical-data-viewer) - - [EVIO Event Viewer on GitHub](https://github.com/JeffersonLab/JEventViewer) - ------------------------------ - -# **Documentation** - ----------------------------- - -Documentation on GitHub: +## **C/C++ Library** -* [All Documentation](https://jeffersonlab.github.io/evio) +To build C/C++ code from this repository: -Documentation on the home page: + git clone https://github.com/JeffersonLab/evio/ + cd evio; mkdir build + cmake -S . -B build + cmake --build build --parallel -* [User's Guide PDF](https://coda.jlab.org/drupal/content/evio-60-users-guide) -* [Javadoc for Java Library](https://coda.jlab.org/drupal/content/evio-60-javadoc) -* [Doxygen for C Library](https://coda.jlab.org/drupal/content/evio-60-doxygen-c) -* [Doxygen for C++ Libary](https://coda.jlab.org/drupal/content/evio-60-doxygen-c-0) +Note that during the cmake configure step (first of two `cmake` commands above), one can +toggle the following special flags: ----------------------------- +* `C_ONLY` : build C lib only, skip C++ (default `-DC_ONLY=0`) +* `MAKE_EXAMPLES`: build example/test programs (default `-DMAKE_EXAMPLES=0`) +* `USE_FILESYSTEMLIB`: ue C++17 instead of Boost (default `-DUSE_FILESYSTEMLIB=0`) +* `DISRUPTOR_FETCH`: allow CMake to download Disruptor if not found (default `-DDISRUPTOR_FETCH=1`) -# **C Library** +One can still also use `scons` instead of cmake to build the evio C/C++ library, though this feature +will not be supported in future releases. ----------------------------- -The C library is called libevio. -It is a library with limited capabilities. In the past, this was acceptable because the evio -format was fairly simple. However, as its complexity has greatly expanded in this version, the C library will -be of very limited usefullness unless one is knowledgeable about all the intricacies of the format. +### Prerequisites -To compile it, follow the directions below for the C++ compilation which will include the C as well. -The C++ library is much more extensive in scope. -Having said that, the C library and executables can be compiled without any C++. This can be done in 2 ways: +C++ 17 or higher, `cmake`, `lz4`, `boost_system`, `boost_thread`, and `boost_chrono`. If LZ4 is not +already configured, it can be installed from [LZ4 on github](https://github.com/lz4/lz4). The boost +libraries are typically system-specific. - scons --C -or +## **Java Library** - mkdir build - cd build - cmake .. –DC_ONLY=1 +The Java version of evio (internally `org.jlab.coda.jevio`) can also be used for reading & writing +`.evio` format files. A "fat" jar file with all dependencies is included in the `java/jars` folder, +which should be all most users need to run and execute Java code utilizing the evio library. +Java 17 is the default version used, however the Java evio library should be compatible with all +java versions 8 and higher (note this has not been rigorously tested). If one wants to create a +new jar file for any reason (e.g. to modify Java versions), do: ----------------------------- + git clone https://github.com/JeffersonLab/evio/ + cd evio + mvn clean install -# **C++ Library** +### Prerequisites ----------------------------- -The C++ library is called libeviocc. -The current C++ evio library is entirely different from the previous version (5.2) as it has been ported -from the Java code. This was done for a number of reasons. -First, a much more comprehensive C++ library was desired than was currently existing. -Second, it needed major, new capabilities such as being able to (un)compress data. -Third, it had to use a new format developed from the merging of Java evio version 4 and the Java HIPO library. -Finally, the author and maintainer of the previous code was no longer working at Jefferson Lab. -The simplest solution was to port the well-tested Java code which avoided having to redesign complex software -from scratch. C++ evio is supported on both the MacOS and Linux platforms. C++ version 11 is used, -and gcc version 5 or higher is required. +Requires Maven (`mvn`) and an installation of Java on your system. +**Running on "ifarm" at JLab will not work unless you install java yourself**. Note that the default java versions on the farm will be too old to +work. See downloads from [OpenJDK](https://openjdk.org/install/) or [Oracle](https://www.oracle.com/java/technologies/javase/jdk17-archive-downloads.html). ----------------------------- -## **Prerequisites** - - -### Disruptor - - -Evio depends upon the Disruptor-cpp software package available from a fork of the original package at github at - -[C++ Disruptor Fork on Github](https://github.com/JeffersonLab/Disruptor-cpp) - -In terms of functionality, it is an ingenious, ultrafast ring buffer which was initially developed in Java -and then ported to C++. It’s extremely useful when splitting work among multiple threads and then recombining it. -To build it, do this on the Mac: - - git clone https://github.com/JeffersonLab/Disruptor-cpp.git - cd Disruptor-cpp - mkdir build - cd build - cmake .. - make - setenv DISRUPTOR_CPP_HOME <../> - -If using Jefferson Lab’s Redhat Enterprise 7 do: - - git clone https://github.com/JeffersonLab/Disruptor-cpp.git - cd Disruptor-cpp - mkdir build - cd build - cmake .. -DCMAKE_C_COMPILER=/apps/gcc/5.3.0/bin/gcc -DCMAKE_CXX_COMPILER=/apps/gcc/5.3.0/bin/g++ - make - setenv DISRUPTOR_CPP_HOME <../> - -Note that it requires GCC 5.0 / Clang 3.8 / C++14 or newer and boost. -Its shared library must be installed where evio can find it. -If not compiling on Jefferson Lab’s RHEL7, either your default compilers must meet this criteria -or you must specify the proper ones on the cmake command line. - - -### Boost - - -Besides the disruptor library, evio requires the boost libraries: boost_system, boost_thread, and boost_chrono. - - -### Lz4 - - -Finally, evio depends on the lz4 library for compressing data in the lz4 and gzip formats. -If it isn’t already available on your machine, it can be obtained from the -[lz4 repository on github](https://github.com/lz4/lz4): - - git clone https://github.com/lz4/lz4.git - cd lz4 - make - make install - - ------------------------------ -## **Building** - - -There are 2 different methods to build the C++ library and executables. -The first uses scons, a Python-based build software package which is available at https://scons.org. -The second uses cmake and make. Also, be sure you’ve set the DISRUPTOR_CPP_HOME environmental variable. - - -### Scons - - -To get a listing of all the local options available to the scons command, -run _**scons -h**_ in the top-level directory to get this output: - - -D build from subdirectory of package - local scons OPTIONS: - --C compile C code only - --dbg compile with debug flag - --32bits compile 32bit libs & executables on 64bit system - --prefix= use base directory when doing install - --incdir= copy header files to directory when doing install - --libdir= copy library files to directory when doing install - --bindir= copy binary files to directory when doing install - install install libs, headers, and binaries - install -c uninstall libs, headers, and binaries - doc create doxygen and javadoc (in ./doc) - undoc remove doxygen and javadoc (in ./doc) - tar create tar file (in ./tar) - - Use scons -H for help about command-line options. - - -Although fairly self-explanatory, if on Jefferson Lab's CUE system with Redhat 7, executing: - - use gcc/5.3.0 - cd - scons install or - scons –-prefix=/my/installation/directory install - -will compile and install all the code. -Note that for C/C++, only Linux and Darwin (Mac OSX) operating systems are supported. -By default, the libraries and executables are placed into the _**$CODA/[arch]/lib**_ and _**bin**_ subdirectories -(eg. Linux-x86_64/lib). If the command line options -–prefix, --incdir, --libdir, or –bindir are used, they take priority. -Be sure to change your LD_LIBRARY_PATH environmental variable to include the correct lib directory. - - -To compile a debug version, execute: - - scons install --dbg - - -### Cmake - - -**Preferable to scons, cmake is the best way to compile evio.** Experience shows that cmake is -simpler, more powerful and more complete than scons. If the disruptor library is not -immediately accessible, cmake can automatically fetch and build it. -Cmake uses the included CMakeLists.txt file. - -To build the C and C++ libraries and executables, start with: - - cd - mkdir build - cd build - -At this point, several choices can be made. - -Evio will fetch and compile its own copy of the Disruptor if neither of the environmental variables -pointing to an existing Disruptor repository, DISRUPTOR_CPP_DIR or DISRUPTOR_CPP_HOME, is defined. -If an existing repository can be found, it will use the includes and libs found there. Simply do: - - cmake .. -To define the Disruptor location on the command line: +# **Useful Links** - cmake .. -DDISRUPTOR_CPP_HOME= or - cmake .. -DDISRUPTOR_CPP_DIR= - -To force Disruptor to be used from an existing local directory: - - cmake .. -DDISRUPTOR_ALLOW_FETCH=OFF -DDISRUPTOR_CPP_HOME= - -When using C++17 and later, #include can be used to access the routines in std::filesystem to deal with the local file system. This can be turned on by defining USE_FILESYSTEMLIB when running cmake. - - cmake .. -DUSE_FILESYSTEMLIB=ON … - -In addition, one can create evioConfig.cmake during install. This allows users to simply use find_package(evio) in their CMake setups. - -If you happen to be building on an older version of linux, such as redhat 7, you’ll need to make sure your g++ is at least version 5. This may look like: - - cmake .. -DCMAKE_C_COMPILER=/apps/gcc/5.3.0/bin/gcc - -DCMAKE_CXX_COMPILER=/apps/gcc/5.3.0/bin/g++ … - -To build only C code, place –DC_ONLY=1 on the cmake command line: - - cmake .. -DC_ONLY=1 … - -To build all the examples as well as libs and programs, place –DMAKE_EXAMPLES=1 on the cmake command line. - - cmake .. -DMAKE_EXAMPLES=1 … - -One can also define the directory in which to install the resulting libs, includes and executables upon compilation: - - cmake .. -DINSTALL= … - -So, for compiling C++ code while fetching the Disruptor and making examples, one might do something like: - - cmake .. -DMAKE_EXAMPLES=1 - -The final step is to call make: - - make -j 4 install - -To uninstall simply do: - - make uninstall - - - - ------------------------------- - -# **Java** - ------------------------------- - - -The current Java evio package, org.jlab.coda.jevio, was originally written by Dr. Dave Heddle of CNU -and was graciously given to the JLAB DAQ group for maintenance and continued development. -A large amount of additional work has been done since that time. As previously mentioned, -evio now uses a new format developed from the merging of evio version 4 and the HIPO library. -The code will compile using Java version 8 or later. - -The jar files necessary to compile an evio jar file are in the java/jars directory. -They are compiled with Java 8. In addition, there are 2 subdirectories: - - * java8, which contains all such jars compiled with Java 8, and - * java15 which contains all jars compiled with Java 15. - -If a jar file is not available in Java 15 use the Java 8 version. - -A pre-compiled _**jevio-6.0.jar**_ file is found in each of these subdirectories. -Using these allows the user to skip over all the following compilation instructions. - - ------------------------------ -## **Prerequisites** - - -### Disruptor - - -Evio depends upon the LMAX-Exchange/disruptor software package available from github whose fork is at: - -[Original Disruptor Fork on Github](https://github.com/JeffersonLab/disruptor) - -In terms of functionality, it is an ingenious, ultrafast ring buffer which was initially developed -for use the in the commodities exchange markets. It’s extremely useful when splitting work among -multiple threads and then recombining it. Although there are many branches in this git repository, -only 2 branches have had the necessary changes to be compatible with CODA. -These are the master and v3.4 branches. The v3.4 branch should be compiled with Java 8 -(it does not compile with Java 15) while the master requires at least Java 11. - -The disruptor software is provided in the _**java/jars/disruptor-3.4.3.jar**_ file (compiled with Java 8). -However, to generate this file yourself, get the disruptor software package by simply doing the following: - - git clone https://github.com/JeffersonLab/disruptor.git - cd disruptor - git checkout v3.4 - ./gradlew - -The resulting disruptor jar file, disruptor-3.4.3.jar, will be found in the disruptor package’s build/libs subdirectory. - -One can also use the master branch which needs to be compiled with Java version 11 or greater and produces disruptor-4.0.0.jar. -Currently this has been created with java15 and is in the java/jars/java15 directory. Here is how to generate it: - - git clone https://github.com/JeffersonLab/disruptor.git - cd disruptor - ./gradlew - -The resulting jar will be in build/libs as before. - - -### Lz4 - - -A lz4 data compression software is provided in the _**java/jars/lz4-java-1.8.0.jar**_ file (compiled with Java 8). -Although this is available in various versions and locations on the web, one can generate this from its source which is the -[lz4/lz4-java repository on github](https://github.com/lz4/lz4-java): - - git clone https://github.com/lz4/lz4-java.git - cd lz4-java - ant ivy-bootstrap - ant submodule init - ant submodule update - ant - -Generated jar files will be in dist subdirectory. - -Another jar file, AHACompressionAPI.jar, also in the the java/jars directory, is for use in Compressor.java -when using the AHA374 FPGA data compression board for gzip compression in hardware. -This is an effort that never took off since LZ4 compresssion was so much more efficient. -Thus, it may be safely ignored or removed. - - ------------------------------ -## **Building** - - -The java evio uses ant to compile. To get a listing of all the options available to the ant command, run +---------------------------- - ant help +Documentation on GitHub: -in the evio top level directory to get this output: +* [All Documentation](https://jeffersonlab.github.io/evio) +* [User's Guide PDF](https://jeffersonlab.github.io/evio/doc-6.0/users_guide/evio_Users_Guide.pdf) +* [EVIO Data Format Reference](https://jeffersonlab.github.io/evio/doc-6.0/format_guide/evio_Formats.pdf) - help: - [echo] Usage: ant [ant options] [target2 | target3 | ...] - - [echo] targets: - [echo] help - print out usage - [echo] env - print out build file variables' values - [echo] compile - compile java files - [echo] clean - remove class files - [echo] cleanall - remove all generated files - [echo] jar - compile and create jar file - [echo] install - create jar file and install into 'prefix' - [echo] if given on command line by -Dprefix=dir', - [echo] else install into CODA if defined - [echo] uninstall - remove jar file previously installed into 'prefix' - [echo] if given on command line by -Dprefix=dir', - [echo] else installed into CODA if defined - [echo] all - clean, compile and create jar file - [echo] javadoc - create javadoc documentation - [echo] developdoc - create javadoc documentation for developer - [echo] undoc - remove all javadoc documentation - [echo] prepare - create necessary directories +Software Library Documentation: +* [Javadoc for Java Library](https://jeffersonlab.github.io/evio/doc-6.0/javadoc/index.html) +* [Doxygen for C Library](https://jeffersonlab.github.io/evio/doc-6.0/doxygen/C/html/index.html) +* [Doxygen for C++ Libary](https://jeffersonlab.github.io/evio/doc-6.0/doxygen/CC/html/index.html) -Although this is fairly self-explanatory, executing _**ant**_ is the same as ant compile. -That will compile all the java. All compiled code is placed in the generated _**jbuild**_ directory. -If the user wants a jar file, execute +Other Links: +* [EVIO Event Viewer on GitHub](https://github.com/JeffersonLab/JEventViewer) - ant jar -to place the resulting file in the _**jbuild/lib**_ directory. -The java command in the user’s path will be the one used to do the compilation. +The EVIO-6 data format is closely related to the HIPO data format, following the same data +framework to the record level. Beyond this level, they differ in the way data is stored as +well as the software used to read and write to these respective `.evio` and `.hipo` formats. +More information on the HIPO data format can be found at https://github.com/gavalian/hipo, +or from the CLAS12 Software Project Coordinator. ---------------------------- @@ -403,4 +92,3 @@ The java command in the user’s path will be the one used to do the compilation ---------------------------- For any issues regarding use and copyright, read the [license](LICENSE.txt) file. - diff --git a/SConstruct b/SConstruct index 241ab6021..1fc5bb7bf 100644 --- a/SConstruct +++ b/SConstruct @@ -17,9 +17,435 @@ # Get various python modules import re, sys, os, string, subprocess, SCons.Node.FS from subprocess import Popen, PIPE +from os import sep, symlink + +# Useful python functions (previously in py) +def recursiveDirs(root) : + """Return a list of all subdirectories of root, top down, + including root, but without .svn and . directories""" + return filter( (lambda a : (a.rfind( ".svn")==-1) and \ + (a.rfind( ".Linux")==-1) and \ + (a.rfind( ".SunOS")==-1) and \ + (a.rfind( ".Darwin")==-1) and \ + (a.rfind( ".vxworks")==-1)), [ a[0] for a in os.walk(root)] ) + + + +def unique(list) : + """Remove duplicates from list""" + return dict.fromkeys(list).keys() + + + +def scanFiles(dir, accept=["*.cpp"], reject=[]) : + """Return a list of selected files from a directory & its subdirectories""" + sources = [] + paths = recursiveDirs(dir) + for path in paths : + for pattern in accept : + sources+=glob.glob(path+"/"+pattern) + for pattern in reject : + sources = filter( (lambda a : a.rfind(pattern)==-1 ), sources ) + return unique(sources) + + + +def subdirsContaining(root, patterns): + """Return a list of subdirectories containing files of the given pattern""" + dirs = unique(map(os.path.dirname, scanFiles(root, patterns))) + dirs.sort() + return dirs + + + +################### +# Operating System +################### + +def CheckHas64Bits(context, flags): + """Define configure-type test function to + see if operating system is 64 or 32 bits""" + + context.Message( 'Checking for 64/32 bits ...' ) + lastCCFLAGS = context.env['CCFLAGS'] + lastLFLAGS = context.env['LINKFLAGS'] + context.env.Append(CCFLAGS = flags, LINKFLAGS = flags) + # (C) program to run to check for bits + ret = context.TryRun(""" +#include +int main(int argc, char **argv) { + printf(\"%d\", 8*sizeof(0L)); + return 0; +} +""", '.c') + # Restore original flags + context.env.Replace(CCFLAGS = lastCCFLAGS, LINKFLAGS = lastLFLAGS) + # If program successfully ran ... + if ret[0]: + context.Result(ret[1]) + if ret[1] == '64': + return 64 + return 32 + # Else if program did not run ... + else: + # Don't know if it's a 32 or 64 bit operating system + context.Result('failed') + return 0 + + + +def is64BitMachine(env, platform, machine): + """Determine if machine has 64 or 32 bit architecture""" + if platform == 'Linux' and machine == 'x86_64': + return True + else: + ccflags = '' + if platform == 'SunOS': + ccflags = '-xarch=amd64' + + # Run the test + conf = Configure( env, custom_tests = { 'CheckBits' : CheckHas64Bits } ) + ret = conf.CheckBits(ccflags) + env = conf.Finish() + if ret < 1: + print ('Cannot run test, assume 64 bit system') + return True + elif ret == 64: + # Test shows 64 bit system' + return True + else: + # Test shows 32 bit system' + return False + + + +def configure32bits(env, use32bits, platform, machine): + """Setup environment on 64 bit machine to handle 32 or 64 bit libs and executables.""" + if platform == 'SunOS': + if not use32bits: + if machine == 'sun4u': + env.Append(CCFLAGS = ['-xarch=native64', '-xcode=pic32'], + LINKFLAGS = ['-xarch=native64', '-xcode=pic32']) + else: + env.Append(CCFLAGS = ['-xarch=amd64'], + LINKFLAGS = ['-xarch=amd64']) + + elif platform == 'Darwin': + if not use32bits: + env.Append(CCFLAGS = [], + LINKFLAGS = ['-Wl', '-bind_at_load']) + + elif platform == 'Linux': + if use32bits: + env.Append(CCFLAGS = ['-m32'], LINKFLAGS = ['-m32']) + + return + + + +########################### +# Installation Directories +########################### + +def getInstallationDirs(osname, prefix, incdir, libdir, bindir): + """Determine installation directories.""" + + # Get CODA env variable if any + codaHomeEnv = os.getenv('CODA','') + + # The installation directory is the user-specified "prefix" + # by first choice, "CODA" secondly. + # Any user specified command line installation path overrides default + if (prefix == None) or (prefix == ''): + # prefix not defined try CODA env var + if codaHomeEnv == "": + if (incdir == None) or (libdir == None) or (bindir == None): + print ("\nNeed to define CODA, or use the --prefix option,") + print ("or all the --incdir, --libdir, and --bindir options.\n") + raise SystemExit + else: + prefix = codaHomeEnv + + osDir = prefix + "/" + osname + + # Set our install directories + if incdir != None: + incDir = incdir + archIncDir = incdir + else: + incDir = prefix + '/common/include' + archIncDir = osDir + '/include' + + if libdir != None: + libDir = libdir + else: + libDir = osDir + '/lib' + + if bindir != None: + binDir = bindir + else: + binDir = osDir + '/bin' + + # Return absolute paths in list + return [os.path.abspath(prefix), + os.path.abspath(osDir), + os.path.abspath(incDir), + os.path.abspath(archIncDir), + os.path.abspath(libDir), + os.path.abspath(binDir)] + + + +########################### +# Include File Directories +########################### + +def makeIncludeDirs(includeDir, archIncludeDir, archDir, archIncLocalLink): + """Function to make include directories""" + # + # Create main include dir if it doesn't exist + # + if not os.path.exists(includeDir): + os.makedirs(includeDir) + # Make sure it's a directory (if we didn't create it) + elif not os.path.isdir(includeDir): + print + print ("Error:", includeDir, "is NOT a directory") + print + raise SystemExit + + if includeDir == archIncludeDir: + return + + # + # If an install has never been done, the arch dir needs + # to be made first so the symbolic link can be created. + # + if not os.path.exists(archDir): + try: + os.makedirs(archDir) + except OSError: + return + elif not os.path.isdir(archDir): + print + print ("Error:", archDir, "is NOT a directory") + print + raise SystemExit + + # + # If the architecture include dir does NOT exist, + # make link to include dir + # + if not os.path.exists(archIncludeDir): + # Create symbolic link: symlink(source, linkname) + try: + if (archIncLocalLink == None) or (archIncLocalLink == ''): + symlink(includeDir, archIncludeDir) + else: + symlink(archIncLocalLink, archIncludeDir) + except OSError: + # Failed to create symbolic link, so + # just make it a regular directory + os.makedirs(archIncludeDir) + elif not os.path.isdir(archIncludeDir): + print + print ("Error:", archIncludeDir, "is NOT a directory") + print + raise SystemExit + + return + + + +########### +# JAVA JNI +########### + +def configureJNI(env): + """Configure the given environment for compiling Java Native Interface + c or c++ language files.""" + + # first look for a shell variable called JAVA_HOME + java_base = os.environ.get('JAVA_HOME') + if not java_base: + if sys.platform == 'darwin': + # Apple's OS X has its own special java base directory + java_base = '/System/Library/Frameworks/JavaVM.framework' + else: + # Search for the java compiler + print ("JAVA_HOME environment variable not set. Searching for javac to find jni.h ...") + if not env.get('JAVAC'): + print ("The Java compiler must be installed and in the current path, exiting") + return 0 + jcdir = os.path.dirname(env.WhereIs('javac')) + if not jcdir: + print (" not found, exiting") + return 0 + # assuming the compiler found is in some directory like + # /usr/jdkX.X/bin/javac, java's home directory is /usr/jdkX.X + java_base = os.path.join(jcdir, "..") + print (" found, dir = " + java_base) + + if sys.platform == 'darwin': + # Apple does not use Sun's naming convention + java_headers = [os.path.join(java_base, 'Headers')] + java_libs = [os.path.join(java_base, 'Libraries')] + else: + # linux + java_headers = [os.path.join(java_base, 'include')] + java_libs = [os.path.join(java_base, 'lib')] + # Sun's windows and linux JDKs keep system-specific header + # files in a sub-directory of include + if java_base == '/usr' or java_base == '/usr/local': + # too many possible subdirectories. Just use defaults + java_headers.append(os.path.join(java_headers[0], 'linux')) + java_headers.append(os.path.join(java_headers[0], 'solaris')) + else: + # add all subdirs of 'include'. The system specific headers + # should be in there somewhere + java_headers = recursiveDirs(java_headers[0]) + + # add Java's include and lib directory to the environment + env.Append(CPPPATH = java_headers) + # (only need the headers right now so comment out next line) + #env.Append(LIBPATH = java_libs) + + # add any special platform-specific compilation or linking flags + # (only need the headers right now so comment out next lines) + #if sys.platform == 'darwin': + # env.Append(SHLINKFLAGS = '-dynamiclib -framework JavaVM') + # env['SHLIBSUFFIX'] = '.jnilib' + + # Add extra potentially useful environment variables + #env['JAVA_HOME'] = java_base + #env['JNI_CPPPATH'] = java_headers + #env['JNI_LIBPATH'] = java_libs + + return 1 + + + + +################### +# Documentation +################### + +def generateDocs(env, doCPP=False, doC=False, doJava=False): + """Generate and install generated documentation (doxygen & javadoc).""" + + if doCPP: + print ('Call generateDocs() for C++') + # remove target files so documentation always gets rebuilt + rmcmd = 'rm -fr doc/doxygen/CC' + os.popen(rmcmd).read() + + def docGeneratorCC(target, source, env): + cmd = 'doxygen doc/doxygen/DoxyfileCC' + Popen(cmd, shell=True, env={"TOPLEVEL": "./"}, stdout=None) + return + + docBuildCC = Builder(action = docGeneratorCC) + env.Append(BUILDERS = {'DocGenCC' : docBuildCC}) + + env.Alias('doc', env.DocGenCC(target = ['#/doc/doxygen/CC/html/index.html'], + source = scanFiles("src/libsrc++", accept=["*.cpp", "*.h"]))) + + + if doC: + rmcmd = 'rm -fr doc/doxygen/C' + os.popen(rmcmd).read() + + def docGeneratorC(target, source, env): + cmd = 'doxygen doc/doxygen/DoxyfileC' + Popen(cmd, shell=True, env={"TOPLEVEL": "./"}, stdout=None) + return + + docBuildC = Builder(action = docGeneratorC) + env.Append(BUILDERS = {'DocGenC' : docBuildC}) + + env.Alias('doc', env.DocGenC(target = ['#/doc/doxygen/C/html/index.html'], + source = scanFiles("src/libsrc", accept=["*.c", "*.h"]))) + + + if doJava: + rmcmd = 'rm -fr doc/javadoc' + os.popen(rmcmd).read() + + def docGeneratorJava(target, source, env): + cmd = 'ant javadoc' + os.popen(cmd).read() + return + + docBuildJava = Builder(action = docGeneratorJava) + env.Append(BUILDERS = {'DocGenJava' : docBuildJava}) + + env.Alias('doc', env.DocGenJava(target = ['#/doc/javadoc/index.html'], + source = scanFiles("java/org/jlab/coda", accept=["*.java"]) )) + + return 1 + + -# Get useful python functions we wrote -import coda + +def removeDocs(env): + """Remove all generated documentation (doxygen & javadoc).""" + + def docRemover(target, source, env): + cmd = 'rm -fr doc/javadoc doc/doxygen/CC doc/doxygen/C' + output = os.popen(cmd).read() + return + + docRemoveAll = Builder(action = docRemover) + env.Append(BUILDERS = {'DocRemove' : docRemoveAll}) + + # remove documentation + env.Alias('undoc', env.DocRemove(target = ['#/.undoc'], source = None)) + + return 1 + + + +################### +# Tar file +################### + +def generateTarFile(env, baseName, majorVersion, minorVersion): + """Generate a gzipped tar file of the current directory.""" + + # Function that does the tar. Note that tar on Solaris is different + # (more primitive) than tar on Linux and MacOS. Solaris tar has no -z option + # and the exclude file does not allow wildcards. Thus, stick to Linux for + # creating the tar file. + def tarballer(target, source, env): + dirname = os.path.basename(os.path.abspath('.')) + cmd = 'tar -X tar/tarexclude -C .. -c -z -f ' + str(target[0]) + ' ./' + dirname + pipe = Popen(cmd, shell=True, stdin=PIPE).stdout + return pipe + + # name of tarfile (software package dependent) + tarfile = 'tar/' + baseName + '-' + majorVersion + '.' + minorVersion + '.tgz' + + # Use a Builder instead of env.Tar() since I can't make that work. + # It runs into circular dependencies since we copy tar file to local + # ./tar directory + tarBuild = Builder(action = tarballer) + env.Append(BUILDERS = {'Tarball' : tarBuild}) + env.Alias('tar', env.Tarball(target = tarfile, source = None)) + + return 1 + + +class color: + PURPLE = '\033[95m' + CYAN = '\033[96m' + DARKCYAN = '\033[36m' + BLUE = '\033[94m' + GREEN = '\033[92m' + YELLOW = '\033[93m' + RED = '\033[91m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + END = '\033[0m' # Created files & dirs will have this permission os.umask(0o2) @@ -39,13 +465,13 @@ osname = os.getenv('CODA_OSNAME', platform + '-' + machine) path = os.getenv('PATH', '') ldLibPath = os.getenv('LD_LIBRARY_PATH', '') -if path == '': +if path == '' or path == None: print print ("Error: set PATH environmental variable") print raise SystemExit -if ldLibPath == '': +if ldLibPath == '' or ldLibPath == None: print print ("Warning: LD_LIBRARY_PATH environmental variable not defined") print @@ -62,7 +488,7 @@ else: # For Linux 64 bit x86 machines, the "machine' variable is x86_64, # but for Darwin or Solaris there is no obvious check so run # a configure-type test. -is64bits = coda.is64BitMachine(env, platform, machine) +is64bits = is64BitMachine(env, platform, machine) if is64bits: print ("We're on a 64-bit machine") else: @@ -131,9 +557,9 @@ env = conf.Finish() # location of C++ version of disruptor disruptorHome = os.getenv('DISRUPTOR_CPP_HOME') -if disruptorHome == "": +if disruptorHome == "" or not os.path.exists(str(disruptorHome)): if not onlyC: - print('Disruptor-cpp must be installed by defining DISRUPTOR_CPP_HOME') + print ("Error: set DISRUPTOR_CPP_HOME environmental variable") Exit(1) else: print('Disruptor-cpp = ' + str(disruptorHome)) @@ -151,13 +577,13 @@ if debug: # Compile with -g and add debugSuffix to all executable names env.Append(CCFLAGS = ['-g'], PROGSUFFIX = debugSuffix) -# code for newlib++ written in C++11 -env.Append(CXXFLAGS = ['-std=c++11']) +# code for newlib++ written in C++17 +env.Append(CXXFLAGS = ['-std=c++17']) # Take care of 64/32 bit issues if is64bits: # Setup 64 bit machine to compile either 32 or 64 bit libs & executables - coda.configure32bits(env, use32bits, platform, machine) + configure32bits(env, use32bits, platform, machine) elif not use32bits: use32bits = True @@ -214,8 +640,8 @@ if (incdir == None): # If we going to install ... if 'install' in COMMAND_LINE_TARGETS: # Determine installation directories - print ("\Call getInstallationDirs with prefix = ", prefix) - installDirs = coda.getInstallationDirs(osname, prefix, incdir, libdir, bindir) + print ("\nCall getInstallationDirs with prefix = ", prefix) + installDirs = getInstallationDirs(osname, prefix, incdir, libdir, bindir) mainInstallDir = installDirs[0] osDir = installDirs[1] @@ -225,7 +651,7 @@ if 'install' in COMMAND_LINE_TARGETS: binInstallDir = installDirs[5] # Create the include directories (make symbolic link if possible) - coda.makeIncludeDirs(incInstallDir, archIncInstallDir, osDir, archIncLocalLink) + makeIncludeDirs(incInstallDir, archIncInstallDir, osDir, archIncLocalLink) print ('Main install dir = '+mainInstallDir) print ('bin install dir = '+binInstallDir) @@ -250,12 +676,12 @@ Help('install -c uninstall libs, headers, and binaries\n') if 'doc' in COMMAND_LINE_TARGETS: if onlyC: - coda.generateDocs(env, False, True) + generateDocs(env, False, True) else: - coda.generateDocs(env, True, True) + generateDocs(env, True, True) if 'undoc' in COMMAND_LINE_TARGETS: - coda.removeDocs(env) + removeDocs(env) # use "doc" on command line to create tar file @@ -270,7 +696,7 @@ Help('undoc remove doxygen and javadoc (in ./doc)\n') ######################### if 'tar' in COMMAND_LINE_TARGETS: - coda.generateTarFile(env, 'evio', versionMajor, versionMinor) + generateTarFile(env, 'evio', versionMajor, versionMinor) # use "tar" on command line to create tar file Help('tar create tar file (in ./tar)\n') @@ -288,8 +714,11 @@ env.SConscript('src/libsrc/SConscript', variant_dir='src/libsrc/'+archDir, dup if not onlyC: env.SConscript('src/libsrc++/SConscript', variant_dir='src/libsrc++/'+archDir, duplicate=0) - #env.SConscript('src/execsrc/SConscript', variant_dir='src/execsrc/'+archDir, duplicate=0) - #env.SConscript('src/examples/SConscript', variant_dir='src/examples/'+archDir, duplicate=0) - env.SConscript('src/test/SConscript', variant_dir='src/test/'+archDir, duplicate=0) + env.SConscript('src/test/cpp/SConscript', variant_dir='src/test/'+archDir, duplicate=0) + +env.SConscript('src/test/c/SConscript', variant_dir='src/testC/'+archDir, duplicate=0) + +print (color.BOLD + color.RED + "WARNING: using scons for EVIO C/C++ libraries is deprecated! \n"\ + " option to build via `scons install` will be removed in future updates \n"\ + " (use cmake instead)" + color.END) -env.SConscript('src/testC/SConscript', variant_dir='src/testC/'+archDir, duplicate=0) diff --git a/coda.py b/coda.py deleted file mode 100644 index 78e863551..000000000 --- a/coda.py +++ /dev/null @@ -1,428 +0,0 @@ -#################################################### -# Python functions to support CODA scons build files -#################################################### - -import re, sys, glob, os, string, subprocess -from os import sep, symlink -from subprocess import Popen, PIPE - -from SCons.Script import Configure -from SCons.Builder import Builder - -################ -# File handling -################ - -def recursiveDirs(root) : - """Return a list of all subdirectories of root, top down, - including root, but without .svn and . directories""" - return filter( (lambda a : (a.rfind( ".svn")==-1) and \ - (a.rfind( ".Linux")==-1) and \ - (a.rfind( ".SunOS")==-1) and \ - (a.rfind( ".Darwin")==-1) and \ - (a.rfind( ".vxworks")==-1)), [ a[0] for a in os.walk(root)] ) - - - -def unique(list) : - """Remove duplicates from list""" - return dict.fromkeys(list).keys() - - - -def scanFiles(dir, accept=["*.cpp"], reject=[]) : - """Return a list of selected files from a directory & its subdirectories""" - sources = [] - paths = recursiveDirs(dir) - for path in paths : - for pattern in accept : - sources+=glob.glob(path+"/"+pattern) - for pattern in reject : - sources = filter( (lambda a : a.rfind(pattern)==-1 ), sources ) - return unique(sources) - - - -def subdirsContaining(root, patterns): - """Return a list of subdirectories containing files of the given pattern""" - dirs = unique(map(os.path.dirname, scanFiles(root, patterns))) - dirs.sort() - return dirs - - - -################### -# Operating System -################### - -def CheckHas64Bits(context, flags): - """Define configure-type test function to - see if operating system is 64 or 32 bits""" - - context.Message( 'Checking for 64/32 bits ...' ) - lastCCFLAGS = context.env['CCFLAGS'] - lastLFLAGS = context.env['LINKFLAGS'] - context.env.Append(CCFLAGS = flags, LINKFLAGS = flags) - # (C) program to run to check for bits - ret = context.TryRun(""" -#include -int main(int argc, char **argv) { - printf(\"%d\", 8*sizeof(0L)); - return 0; -} -""", '.c') - # Restore original flags - context.env.Replace(CCFLAGS = lastCCFLAGS, LINKFLAGS = lastLFLAGS) - # If program successfully ran ... - if ret[0]: - context.Result(ret[1]) - if ret[1] == '64': - return 64 - return 32 - # Else if program did not run ... - else: - # Don't know if it's a 32 or 64 bit operating system - context.Result('failed') - return 0 - - - -def is64BitMachine(env, platform, machine): - """Determine if machine has 64 or 32 bit architecture""" - if platform == 'Linux' and machine == 'x86_64': - return True - else: - ccflags = '' - if platform == 'SunOS': - ccflags = '-xarch=amd64' - - # Run the test - conf = Configure( env, custom_tests = { 'CheckBits' : CheckHas64Bits } ) - ret = conf.CheckBits(ccflags) - env = conf.Finish() - if ret < 1: - print ('Cannot run test, assume 64 bit system') - return True - elif ret == 64: - # Test shows 64 bit system' - return True - else: - # Test shows 32 bit system' - return False - - - -def configure32bits(env, use32bits, platform, machine): - """Setup environment on 64 bit machine to handle 32 or 64 bit libs and executables.""" - if platform == 'SunOS': - if not use32bits: - if machine == 'sun4u': - env.Append(CCFLAGS = ['-xarch=native64', '-xcode=pic32'], - LINKFLAGS = ['-xarch=native64', '-xcode=pic32']) - else: - env.Append(CCFLAGS = ['-xarch=amd64'], - LINKFLAGS = ['-xarch=amd64']) - - elif platform == 'Darwin': - if not use32bits: - env.Append(CCFLAGS = [], - LINKFLAGS = ['-Wl', '-bind_at_load']) - - elif platform == 'Linux': - if use32bits: - env.Append(CCFLAGS = ['-m32'], LINKFLAGS = ['-m32']) - - return - - - -########################### -# Installation Directories -########################### - -def getInstallationDirs(osname, prefix, incdir, libdir, bindir): - """Determine installation directories.""" - - # Get CODA env variable if any - codaHomeEnv = os.getenv('CODA','') - - # The installation directory is the user-specified "prefix" - # by first choice, "CODA" secondly. - # Any user specified command line installation path overrides default - if (prefix == None) or (prefix == ''): - # prefix not defined try CODA env var - if codaHomeEnv == "": - if (incdir == None) or (libdir == None) or (bindir == None): - print ("\nNeed to define CODA, or use the --prefix option,") - print ("or all the --incdir, --libdir, and --bindir options.\n") - raise SystemExit - else: - prefix = codaHomeEnv - - osDir = prefix + "/" + osname - - # Set our install directories - if incdir != None: - incDir = incdir - archIncDir = incdir - else: - incDir = prefix + '/common/include' - archIncDir = osDir + '/include' - - if libdir != None: - libDir = libdir - else: - libDir = osDir + '/lib' - - if bindir != None: - binDir = bindir - else: - binDir = osDir + '/bin' - - # Return absolute paths in list - return [os.path.abspath(prefix), - os.path.abspath(osDir), - os.path.abspath(incDir), - os.path.abspath(archIncDir), - os.path.abspath(libDir), - os.path.abspath(binDir)] - - - -########################### -# Include File Directories -########################### - -def makeIncludeDirs(includeDir, archIncludeDir, archDir, archIncLocalLink): - """Function to make include directories""" - # - # Create main include dir if it doesn't exist - # - if not os.path.exists(includeDir): - os.makedirs(includeDir) - # Make sure it's a directory (if we didn't create it) - elif not os.path.isdir(includeDir): - print - print ("Error:", includeDir, "is NOT a directory") - print - raise SystemExit - - if includeDir == archIncludeDir: - return - - # - # If an install has never been done, the arch dir needs - # to be made first so the symbolic link can be created. - # - if not os.path.exists(archDir): - try: - os.makedirs(archDir) - except OSError: - return - elif not os.path.isdir(archDir): - print - print ("Error:", archDir, "is NOT a directory") - print - raise SystemExit - - # - # If the architecture include dir does NOT exist, - # make link to include dir - # - if not os.path.exists(archIncludeDir): - # Create symbolic link: symlink(source, linkname) - try: - if (archIncLocalLink == None) or (archIncLocalLink == ''): - symlink(includeDir, archIncludeDir) - else: - symlink(archIncLocalLink, archIncludeDir) - except OSError: - # Failed to create symbolic link, so - # just make it a regular directory - os.makedirs(archIncludeDir) - elif not os.path.isdir(archIncludeDir): - print - print ("Error:", archIncludeDir, "is NOT a directory") - print - raise SystemExit - - return - - - -########### -# JAVA JNI -########### - -def configureJNI(env): - """Configure the given environment for compiling Java Native Interface - c or c++ language files.""" - - # first look for a shell variable called JAVA_HOME - java_base = os.environ.get('JAVA_HOME') - if not java_base: - if sys.platform == 'darwin': - # Apple's OS X has its own special java base directory - java_base = '/System/Library/Frameworks/JavaVM.framework' - else: - # Search for the java compiler - print ("JAVA_HOME environment variable not set. Searching for javac to find jni.h ...") - if not env.get('JAVAC'): - print ("The Java compiler must be installed and in the current path, exiting") - return 0 - jcdir = os.path.dirname(env.WhereIs('javac')) - if not jcdir: - print (" not found, exiting") - return 0 - # assuming the compiler found is in some directory like - # /usr/jdkX.X/bin/javac, java's home directory is /usr/jdkX.X - java_base = os.path.join(jcdir, "..") - print (" found, dir = " + java_base) - - if sys.platform == 'darwin': - # Apple does not use Sun's naming convention - java_headers = [os.path.join(java_base, 'Headers')] - java_libs = [os.path.join(java_base, 'Libraries')] - else: - # linux - java_headers = [os.path.join(java_base, 'include')] - java_libs = [os.path.join(java_base, 'lib')] - # Sun's windows and linux JDKs keep system-specific header - # files in a sub-directory of include - if java_base == '/usr' or java_base == '/usr/local': - # too many possible subdirectories. Just use defaults - java_headers.append(os.path.join(java_headers[0], 'linux')) - java_headers.append(os.path.join(java_headers[0], 'solaris')) - else: - # add all subdirs of 'include'. The system specific headers - # should be in there somewhere - java_headers = recursiveDirs(java_headers[0]) - - # add Java's include and lib directory to the environment - env.Append(CPPPATH = java_headers) - # (only need the headers right now so comment out next line) - #env.Append(LIBPATH = java_libs) - - # add any special platform-specific compilation or linking flags - # (only need the headers right now so comment out next lines) - #if sys.platform == 'darwin': - # env.Append(SHLINKFLAGS = '-dynamiclib -framework JavaVM') - # env['SHLIBSUFFIX'] = '.jnilib' - - # Add extra potentially useful environment variables - #env['JAVA_HOME'] = java_base - #env['JNI_CPPPATH'] = java_headers - #env['JNI_LIBPATH'] = java_libs - - return 1 - - - - -################### -# Documentation -################### - -def generateDocs(env, doCPP=False, doC=False, doJava=False): - """Generate and install generated documentation (doxygen & javadoc).""" - - if doCPP: - print ('Call generateDocs() for C++') - # remove target files so documentation always gets rebuilt - rmcmd = 'rm -fr doc/doxygen/CC' - os.popen(rmcmd).read() - - def docGeneratorCC(target, source, env): - cmd = 'doxygen doc/doxygen/DoxyfileCC' - Popen(cmd, shell=True, env={"TOPLEVEL": "./"}, stdout=None) - return - - docBuildCC = Builder(action = docGeneratorCC) - env.Append(BUILDERS = {'DocGenCC' : docBuildCC}) - - env.Alias('doc', env.DocGenCC(target = ['#/doc/doxygen/CC/html/index.html'], - source = scanFiles("src/libsrc++", accept=["*.cpp", "*.h"]))) - - - if doC: - rmcmd = 'rm -fr doc/doxygen/C' - os.popen(rmcmd).read() - - def docGeneratorC(target, source, env): - cmd = 'doxygen doc/doxygen/DoxyfileC' - Popen(cmd, shell=True, env={"TOPLEVEL": "./"}, stdout=None) - return - - docBuildC = Builder(action = docGeneratorC) - env.Append(BUILDERS = {'DocGenC' : docBuildC}) - - env.Alias('doc', env.DocGenC(target = ['#/doc/doxygen/C/html/index.html'], - source = scanFiles("src/libsrc", accept=["*.c", "*.h"]))) - - - if doJava: - rmcmd = 'rm -fr doc/javadoc' - os.popen(rmcmd).read() - - def docGeneratorJava(target, source, env): - cmd = 'ant javadoc' - os.popen(cmd).read() - return - - docBuildJava = Builder(action = docGeneratorJava) - env.Append(BUILDERS = {'DocGenJava' : docBuildJava}) - - env.Alias('doc', env.DocGenJava(target = ['#/doc/javadoc/index.html'], - source = scanFiles("java/org/jlab/coda", accept=["*.java"]) )) - - return 1 - - - - -def removeDocs(env): - """Remove all generated documentation (doxygen & javadoc).""" - - def docRemover(target, source, env): - cmd = 'rm -fr doc/javadoc doc/doxygen/CC doc/doxygen/C' - output = os.popen(cmd).read() - return - - docRemoveAll = Builder(action = docRemover) - env.Append(BUILDERS = {'DocRemove' : docRemoveAll}) - - # remove documentation - env.Alias('undoc', env.DocRemove(target = ['#/.undoc'], source = None)) - - return 1 - - - -################### -# Tar file -################### - -def generateTarFile(env, baseName, majorVersion, minorVersion): - """Generate a gzipped tar file of the current directory.""" - - # Function that does the tar. Note that tar on Solaris is different - # (more primitive) than tar on Linux and MacOS. Solaris tar has no -z option - # and the exclude file does not allow wildcards. Thus, stick to Linux for - # creating the tar file. - def tarballer(target, source, env): - dirname = os.path.basename(os.path.abspath('.')) - cmd = 'tar -X tar/tarexclude -C .. -c -z -f ' + str(target[0]) + ' ./' + dirname - pipe = Popen(cmd, shell=True, stdin=PIPE).stdout - return pipe - - # name of tarfile (software package dependent) - tarfile = 'tar/' + baseName + '-' + majorVersion + '.' + minorVersion + '.tgz' - - # Use a Builder instead of env.Tar() since I can't make that work. - # It runs into circular dependencies since we copy tar file to local - # ./tar directory - tarBuild = Builder(action = tarballer) - env.Append(BUILDERS = {'Tarball' : tarBuild}) - env.Alias('tar', env.Tarball(target = tarfile, source = None)) - - return 1 diff --git a/CHANGES b/etc/CHANGES similarity index 97% rename from CHANGES rename to etc/CHANGES index 330ba87a7..55cca1b1d 100644 --- a/CHANGES +++ b/etc/CHANGES @@ -1,3 +1,8 @@ +********************************************************************************* +JAVA CODE CHANGE LOG (oldest to newest) +********************************************************************************* + + jevio-4.3.1.jar differences from previous jar : -check lengths when parsing evio so bad values are caught if possible @@ -165,7 +170,7 @@ jevio-5.1.jar ********************************************************************************* -C CODE +C/C++ CODE CHANGE LOG (oldest to newest) ********************************************************************************* evio-4.3.3 diff --git a/etc/README.md b/etc/README.md new file mode 100644 index 000000000..f131cad18 --- /dev/null +++ b/etc/README.md @@ -0,0 +1,11 @@ +# **Miscellaneous folder** + +This folder collects old scripts, documentation, and other misc files from evio versions 1-5. + +| File/Folder | Description +|------------|---------------------------------------------------------------------- +| [java](java) | Collection of Java resources, Xml2evio program, old jar dependency files, and ant build xml. | +| [src](src) | Collection of Java, C/C++ library code. Essentially, these are all old test programs. | +| [CHANGES](CHANGES) | Change log for evio versions 4-5 | +| [cmake_uninstall.cmake.in](cmake_uninstall.cmake.in) | Uninstallation script for evio-5 | +| [setup_java](setup_java) | Setup script for choosing java version (either Java 8 or 15). | diff --git a/etc/README_v6.0.md b/etc/README_v6.0.md new file mode 100644 index 000000000..55cedf6d6 --- /dev/null +++ b/etc/README_v6.0.md @@ -0,0 +1,412 @@ + + +---------------------------- + +# **EVIO 6.0 SOFTWARE PACKAGE** + +This may useful for some historical reference, as it represents a record of commands and +setup used prior to the change to Alma Linux as Jefferson Lab's preferred distro for on-site +resources. + +---------------------------- + +EVIO stands for EVent Input/Output and contains libraries which read & write +data in its own unique format. It was created by the Data Acquisition (DAQ) group +and is now maintained by the Experimental Physics Software and Computing Infrastructure +(EPSCI) group at Thomas Jefferson National Accelerator Facility (Jefferson Lab). +It was originally used in the online DAQ as part of the CODA software, +but is now being used offline and in other applications as well. + +There is a C library as well as an independent C++ library which +is a port from the Java version. There are a few utility programs included. +The Java version is a library with an extensive, user-friendly +interface. There is an additional package containing a GUI for looking at +evio format data at the byte level. + +If you only plan to run C/C++ applications you can skip the Java +installation. If you only plan to use Java applications you can +you can skip the C/C++ installation. + +### **Main evio links:** + + [EVIO Home Page](https://coda.jlab.org/drupal/content/event-io-evio/) + + [EVIO on GitHub (**evio-6.0** branch)](https://github.com/JeffersonLab/evio) + +### **GUI for viewing data in evio format:** + + [EVIO Event Viewer Home Page](https://coda.jlab.org/drupal/content/graphical-data-viewer) + + [EVIO Event Viewer on GitHub](https://github.com/JeffersonLab/JEventViewer) + +----------------------------- + +# **Documentation** + +---------------------------- + +Documentation on GitHub: + +* [All Documentation](https://jeffersonlab.github.io/evio) + +Documentation on the home page: + +* [User's Guide PDF](https://coda.jlab.org/drupal/content/evio-60-users-guide) +* [Javadoc for Java Library](https://coda.jlab.org/drupal/content/evio-60-javadoc) +* [Doxygen for C Library](https://coda.jlab.org/drupal/content/evio-60-doxygen-c) +* [Doxygen for C++ Libary](https://coda.jlab.org/drupal/content/evio-60-doxygen-c-0) + +---------------------------- + +# **C Library** + +---------------------------- +The C library is called libevio. +It is a library with limited capabilities. In the past, this was acceptable because the evio +format was fairly simple. However, as its complexity has greatly expanded in this version, the C library will +be of very limited usefullness unless one is knowledgeable about all the intricacies of the format. + +To compile it, follow the directions below for the C++ compilation which will include the C as well. +The C++ library is much more extensive in scope. +Having said that, the C library and executables can be compiled without any C++. This can be done in 2 ways: + + scons --C +or + + mkdir build + cd build + cmake .. –DC_ONLY=1 + + +---------------------------- + +# **C++ Library** + +---------------------------- +The C++ library is called libeviocc. +The current C++ evio library is entirely different from the previous version (5.2) as it has been ported +from the Java code. This was done for a number of reasons. +First, a much more comprehensive C++ library was desired than was currently existing. +Second, it needed major, new capabilities such as being able to (un)compress data. +Third, it had to use a new format developed from the merging of Java evio version 4 and the Java HIPO library. +Finally, the author and maintainer of the previous code was no longer working at Jefferson Lab. +The simplest solution was to port the well-tested Java code which avoided having to redesign complex software +from scratch. C++ evio is supported on both the MacOS and Linux platforms. C++ version 11 is used, +and gcc version 5 or higher is required. + + +----------------------------- +## **Prerequisites** + + +### Disruptor + + +Evio depends upon the Disruptor-cpp software package available from a fork of the original package at github at + +[C++ Disruptor Fork on Github](https://github.com/JeffersonLab/Disruptor-cpp) + +In terms of functionality, it is an ingenious, ultrafast ring buffer which was initially developed in Java +and then ported to C++. It’s extremely useful when splitting work among multiple threads and then recombining it. +To build it, do this on the Mac: + + git clone https://github.com/JeffersonLab/Disruptor-cpp.git + cd Disruptor-cpp + mkdir build + cd build + cmake .. + make + setenv DISRUPTOR_CPP_HOME <../> + +If using Jefferson Lab’s Redhat Enterprise 7 do: + + git clone https://github.com/JeffersonLab/Disruptor-cpp.git + cd Disruptor-cpp + mkdir build + cd build + cmake .. -DCMAKE_C_COMPILER=/apps/gcc/5.3.0/bin/gcc -DCMAKE_CXX_COMPILER=/apps/gcc/5.3.0/bin/g++ + make + setenv DISRUPTOR_CPP_HOME <../> + +Note that it requires GCC 5.0 / Clang 3.8 / C++14 or newer and boost. +Its shared library must be installed where evio can find it. +If not compiling on Jefferson Lab’s RHEL7, either your default compilers must meet this criteria +or you must specify the proper ones on the cmake command line. + + +### Boost + + +Besides the disruptor library, evio requires the boost libraries: boost_system, boost_thread, and boost_chrono. + + +### Lz4 + + +Finally, evio depends on the lz4 library for compressing data in the lz4 and gzip formats. +If it isn’t already available on your machine, it can be obtained from the +[lz4 repository on github](https://github.com/lz4/lz4): + + git clone https://github.com/lz4/lz4.git + cd lz4 + make + make install + + +----------------------------- +## **Building** + + +There are 2 different methods to build the C++ library and executables. +The first uses scons, a Python-based build software package which is available at https://scons.org. +The second uses cmake and make. Also, be sure you’ve set the DISRUPTOR_CPP_HOME environmental variable. + + +### Scons + + +To get a listing of all the local options available to the scons command, +run _**scons -h**_ in the top-level directory to get this output: + + -D build from subdirectory of package + local scons OPTIONS: + --C compile C code only + --dbg compile with debug flag + --32bits compile 32bit libs & executables on 64bit system + --prefix= use base directory when doing install + --incdir= copy header files to directory when doing install + --libdir= copy library files to directory when doing install + --bindir= copy binary files to directory when doing install + install install libs, headers, and binaries + install -c uninstall libs, headers, and binaries + doc create doxygen and javadoc (in ./doc) + undoc remove doxygen and javadoc (in ./doc) + tar create tar file (in ./tar) + + Use scons -H for help about command-line options. + + +Although fairly self-explanatory, if on Jefferson Lab's CUE system with Redhat 7, executing: + + use gcc/5.3.0 + cd + scons install or + scons –-prefix=/my/installation/directory install + +will compile and install all the code. +Note that for C/C++, only Linux and Darwin (Mac OSX) operating systems are supported. +By default, the libraries and executables are placed into the _**$CODA/[arch]/lib**_ and _**bin**_ subdirectories +(eg. Linux-x86_64/lib). If the command line options +–prefix, --incdir, --libdir, or –bindir are used, they take priority. +Be sure to change your LD_LIBRARY_PATH environmental variable to include the correct lib directory. + + +To compile a debug version, execute: + + scons install --dbg + + +### Cmake + + +**Preferable to scons, cmake is the best way to compile evio.** Experience shows that cmake is +simpler, more powerful and more complete than scons. If the disruptor library is not +immediately accessible, cmake can automatically fetch and build it. +Cmake uses the included CMakeLists.txt file. + +To build the C and C++ libraries and executables, start with: + + cd + mkdir build + cd build + +At this point, several choices can be made. + +Evio will fetch and compile its own copy of the Disruptor if neither of the environmental variables +pointing to an existing Disruptor repository, DISRUPTOR_CPP_DIR or DISRUPTOR_CPP_HOME, is defined. +If an existing repository can be found, it will use the includes and libs found there. Simply do: + + cmake .. + +To define the Disruptor location on the command line: + + cmake .. -DDISRUPTOR_CPP_HOME= or + cmake .. -DDISRUPTOR_CPP_DIR= + +To force Disruptor to be used from an existing local directory: + + cmake .. -DDISRUPTOR_ALLOW_FETCH=OFF -DDISRUPTOR_CPP_HOME= + +When using C++17 and later, #include can be used to access the routines in std::filesystem to deal with the local file system. This can be turned on by defining USE_FILESYSTEMLIB when running cmake. + + cmake .. -DUSE_FILESYSTEMLIB=ON … + +In addition, one can create evioConfig.cmake during install. This allows users to simply use find_package(evio) in their CMake setups. + +If you happen to be building on an older version of linux, such as redhat 7, you’ll need to make sure your g++ is at least version 5. This may look like: + + cmake .. -DCMAKE_C_COMPILER=/apps/gcc/5.3.0/bin/gcc + -DCMAKE_CXX_COMPILER=/apps/gcc/5.3.0/bin/g++ … + +To build only C code, place –DC_ONLY=1 on the cmake command line: + + cmake .. -DC_ONLY=1 … + +To build all the examples as well as libs and programs, place –DMAKE_EXAMPLES=1 on the cmake command line. + + cmake .. -DMAKE_EXAMPLES=1 … + +One can also define the directory in which to install the resulting libs, includes and executables upon compilation: + + cmake .. -DINSTALL= … + +So, for compiling C++ code while fetching the Disruptor and making examples, one might do something like: + + cmake .. -DMAKE_EXAMPLES=1 + +The final step is to call make: + + make -j 4 install + +To uninstall simply do: + + make uninstall + + + + +------------------------------ + +# **Java** + +------------------------------ + + +The current Java evio package, org.jlab.coda.jevio, was originally written by Dr. Dave Heddle of CNU +and was graciously given to the JLAB DAQ group for maintenance and continued development. +A large amount of additional work has been done since that time. As previously mentioned, +evio now uses a new format developed from the merging of evio version 4 and the HIPO library. +The code will compile using Java version 8 or later. + +The jar files necessary to compile an evio jar file are in the java/jars directory. +They are compiled with Java 8. In addition, there are 2 subdirectories: + + * java8, which contains all such jars compiled with Java 8, and + * java15 which contains all jars compiled with Java 15. + +If a jar file is not available in Java 15 use the Java 8 version. + +A pre-compiled _**jevio-6.0.jar**_ file is found in each of these subdirectories. +Using these allows the user to skip over all the following compilation instructions. + + +----------------------------- +## **Prerequisites** + + +### Disruptor + + +Evio depends upon the LMAX-Exchange/disruptor software package available from github whose fork is at: + +[Original Disruptor Fork on Github](https://github.com/JeffersonLab/disruptor) + +In terms of functionality, it is an ingenious, ultrafast ring buffer which was initially developed +for use the in the commodities exchange markets. It’s extremely useful when splitting work among +multiple threads and then recombining it. Although there are many branches in this git repository, +only 2 branches have had the necessary changes to be compatible with CODA. +These are the master and v3.4 branches. The v3.4 branch should be compiled with Java 8 +(it does not compile with Java 15) while the master requires at least Java 11. + +The disruptor software is provided in the _**java/jars/disruptor-3.4.3.jar**_ file (compiled with Java 8). +However, to generate this file yourself, get the disruptor software package by simply doing the following: + + git clone https://github.com/JeffersonLab/disruptor.git + cd disruptor + git checkout v3.4 + ./gradlew + +The resulting disruptor jar file, disruptor-3.4.3.jar, will be found in the disruptor package’s build/libs subdirectory. + +One can also use the master branch which needs to be compiled with Java version 11 or greater and produces disruptor-4.0.0.jar. +Currently this has been created with java15 and is in the java/jars/java15 directory. Here is how to generate it: + + git clone https://github.com/JeffersonLab/disruptor.git + cd disruptor + ./gradlew + +The resulting jar will be in build/libs as before. + + +### Lz4 + + +A lz4 data compression software is provided in the _**java/jars/lz4-java-1.8.0.jar**_ file (compiled with Java 8). +Although this is available in various versions and locations on the web, one can generate this from its source which is the +[lz4/lz4-java repository on github](https://github.com/lz4/lz4-java): + + git clone https://github.com/lz4/lz4-java.git + cd lz4-java + ant ivy-bootstrap + ant submodule init + ant submodule update + ant + +Generated jar files will be in dist subdirectory. + +Another jar file, AHACompressionAPI.jar, also in the the java/jars directory, is for use in Compressor.java +when using the AHA374 FPGA data compression board for gzip compression in hardware. +This is an effort that never took off since LZ4 compresssion was so much more efficient. +Thus, it may be safely ignored or removed. + + +----------------------------- +## **Building** + + +The java evio uses ant to compile. To get a listing of all the options available to the ant command, run + + ant help + +in the evio top level directory to get this output: + + help: + [echo] Usage: ant [ant options] [target2 | target3 | ...] + + [echo] targets: + [echo] help - print out usage + [echo] env - print out build file variables' values + [echo] compile - compile java files + [echo] clean - remove class files + [echo] cleanall - remove all generated files + [echo] jar - compile and create jar file + [echo] install - create jar file and install into 'prefix' + [echo] if given on command line by -Dprefix=dir', + [echo] else install into CODA if defined + [echo] uninstall - remove jar file previously installed into 'prefix' + [echo] if given on command line by -Dprefix=dir', + [echo] else installed into CODA if defined + [echo] all - clean, compile and create jar file + [echo] javadoc - create javadoc documentation + [echo] developdoc - create javadoc documentation for developer + [echo] undoc - remove all javadoc documentation + [echo] prepare - create necessary directories + + +Although this is fairly self-explanatory, executing _**ant**_ is the same as ant compile. +That will compile all the java. All compiled code is placed in the generated _**jbuild**_ directory. +If the user wants a jar file, execute + + ant jar + +to place the resulting file in the _**jbuild/lib**_ directory. +The java command in the user’s path will be the one used to do the compilation. + +---------------------------- + +# **Copyright** + +---------------------------- + +For any issues regarding use and copyright, read the [license](LICENSE.txt) file. + diff --git a/cmake_uninstall.cmake.in b/etc/cmake_uninstall.cmake.in similarity index 100% rename from cmake_uninstall.cmake.in rename to etc/cmake_uninstall.cmake.in diff --git a/build.xml b/etc/java/build.xml similarity index 100% rename from build.xml rename to etc/java/build.xml diff --git a/java/org/jlab/coda/jevio/apps/Xml2evio.java b/etc/java/org/jlab/coda/jevio/apps/Xml2evio.java similarity index 100% rename from java/org/jlab/coda/jevio/apps/Xml2evio.java rename to etc/java/org/jlab/coda/jevio/apps/Xml2evio.java diff --git a/java/jars/disruptor-3.4.3.jar b/etc/java/org/jlab/jars/disruptor-3.4.3.jar similarity index 100% rename from java/jars/disruptor-3.4.3.jar rename to etc/java/org/jlab/jars/disruptor-3.4.3.jar diff --git a/java/jars/java15/disruptor-4.0.0.jar b/etc/java/org/jlab/jars/java15/disruptor-4.0.0.jar similarity index 100% rename from java/jars/java15/disruptor-4.0.0.jar rename to etc/java/org/jlab/jars/java15/disruptor-4.0.0.jar diff --git a/java/jars/java15/jevio-6.0.jar b/etc/java/org/jlab/jars/java15/jevio-6.0.jar similarity index 100% rename from java/jars/java15/jevio-6.0.jar rename to etc/java/org/jlab/jars/java15/jevio-6.0.jar diff --git a/java/jars/java15/lz4-java.1.8.0.jar b/etc/java/org/jlab/jars/java15/lz4-java.1.8.0.jar similarity index 100% rename from java/jars/java15/lz4-java.1.8.0.jar rename to etc/java/org/jlab/jars/java15/lz4-java.1.8.0.jar diff --git a/java/jars/java8/disruptor-3.4.3.jar b/etc/java/org/jlab/jars/java8/disruptor-3.4.3.jar similarity index 100% rename from java/jars/java8/disruptor-3.4.3.jar rename to etc/java/org/jlab/jars/java8/disruptor-3.4.3.jar diff --git a/java/jars/java8/jevio-6.0.jar b/etc/java/org/jlab/jars/java8/jevio-6.0.jar similarity index 100% rename from java/jars/java8/jevio-6.0.jar rename to etc/java/org/jlab/jars/java8/jevio-6.0.jar diff --git a/java/jars/java8/lz4-java.1.8.0.jar b/etc/java/org/jlab/jars/java8/lz4-java.1.8.0.jar similarity index 100% rename from java/jars/java8/lz4-java.1.8.0.jar rename to etc/java/org/jlab/jars/java8/lz4-java.1.8.0.jar diff --git a/java/jars/lz4-java.1.8.0.jar b/etc/java/org/jlab/jars/lz4-java.1.8.0.jar similarity index 100% rename from java/jars/lz4-java.1.8.0.jar rename to etc/java/org/jlab/jars/lz4-java.1.8.0.jar diff --git a/setup_java b/etc/setup_java similarity index 100% rename from setup_java rename to etc/setup_java diff --git a/src/execsrc/README b/etc/src/execsrc/README similarity index 100% rename from src/execsrc/README rename to etc/src/execsrc/README diff --git a/src/execsrc/SConscript b/etc/src/execsrc/SConscript similarity index 100% rename from src/execsrc/SConscript rename to etc/src/execsrc/SConscript diff --git a/src/testC/SConscript b/etc/src/test/c/SConscript similarity index 100% rename from src/testC/SConscript rename to etc/src/test/c/SConscript diff --git a/src/testC/evReadPipe.c b/etc/src/test/c/evReadPipe.c similarity index 100% rename from src/testC/evReadPipe.c rename to etc/src/test/c/evReadPipe.c diff --git a/src/testC/evTestFile.c b/etc/src/test/c/evTestFile.c similarity index 100% rename from src/testC/evTestFile.c rename to etc/src/test/c/evTestFile.c diff --git a/src/testC/evWritePipe.c b/etc/src/test/c/evWritePipe.c similarity index 100% rename from src/testC/evWritePipe.c rename to etc/src/test/c/evWritePipe.c diff --git a/src/testC/evtest.c b/etc/src/test/c/evtest.c similarity index 100% rename from src/testC/evtest.c rename to etc/src/test/c/evtest.c diff --git a/src/testC/evtest2.c b/etc/src/test/c/evtest2.c similarity index 100% rename from src/testC/evtest2.c rename to etc/src/test/c/evtest2.c diff --git a/src/testC/evtest3.c b/etc/src/test/c/evtest3.c similarity index 100% rename from src/testC/evtest3.c rename to etc/src/test/c/evtest3.c diff --git a/src/testC/evtestAppend.c b/etc/src/test/c/evtestAppend.c similarity index 100% rename from src/testC/evtestAppend.c rename to etc/src/test/c/evtestAppend.c diff --git a/src/testC/evtestBuf.c b/etc/src/test/c/evtestBuf.c similarity index 100% rename from src/testC/evtestBuf.c rename to etc/src/test/c/evtestBuf.c diff --git a/src/testC/evtestBuf2.c b/etc/src/test/c/evtestBuf2.c similarity index 100% rename from src/testC/evtestBuf2.c rename to etc/src/test/c/evtestBuf2.c diff --git a/src/testC/evtestRead.c b/etc/src/test/c/evtestRead.c similarity index 100% rename from src/testC/evtestRead.c rename to etc/src/test/c/evtestRead.c diff --git a/src/testC/evtestRio.c b/etc/src/test/c/evtestRio.c similarity index 100% rename from src/testC/evtestRio.c rename to etc/src/test/c/evtestRio.c diff --git a/src/testC/evtestSock.c b/etc/src/test/c/evtestSock.c similarity index 100% rename from src/testC/evtestSock.c rename to etc/src/test/c/evtestSock.c diff --git a/src/testC/evtestWriteFile.c b/etc/src/test/c/evtestWriteFile.c similarity index 100% rename from src/testC/evtestWriteFile.c rename to etc/src/test/c/evtestWriteFile.c diff --git a/src/testC/splitNameTest.c b/etc/src/test/c/splitNameTest.c similarity index 100% rename from src/testC/splitNameTest.c rename to etc/src/test/c/splitNameTest.c diff --git a/src/testC/splitTest.c b/etc/src/test/c/splitTest.c similarity index 100% rename from src/testC/splitTest.c rename to etc/src/test/c/splitTest.c diff --git a/src/test/CompactBuilder_Test.cpp b/etc/src/test/cpp/CompactBuilder_Test.cpp similarity index 100% rename from src/test/CompactBuilder_Test.cpp rename to etc/src/test/cpp/CompactBuilder_Test.cpp diff --git a/src/test/CompactReaderAddSubtractTest.cpp b/etc/src/test/cpp/CompactReaderAddSubtractTest.cpp similarity index 100% rename from src/test/CompactReaderAddSubtractTest.cpp rename to etc/src/test/cpp/CompactReaderAddSubtractTest.cpp diff --git a/src/test/CompactReaderBugTest.cpp b/etc/src/test/cpp/CompactReaderBugTest.cpp similarity index 100% rename from src/test/CompactReaderBugTest.cpp rename to etc/src/test/cpp/CompactReaderBugTest.cpp diff --git a/src/test/CompositeTester.cpp b/etc/src/test/cpp/CompositeTester.cpp similarity index 100% rename from src/test/CompositeTester.cpp rename to etc/src/test/cpp/CompositeTester.cpp diff --git a/src/test/DictTest.cpp b/etc/src/test/cpp/DictTest.cpp similarity index 99% rename from src/test/DictTest.cpp rename to etc/src/test/cpp/DictTest.cpp index eae1c7e59..78223ead3 100644 --- a/src/test/DictTest.cpp +++ b/etc/src/test/cpp/DictTest.cpp @@ -272,8 +272,8 @@ namespace evio { uint16_t tag = 0; uint8_t num = 0; - bool gotTag = dict.getTag("second(1)", &tag); - bool gotNum = dict.getNum("second(1)", &num); + // bool gotTag = dict.getTag("second(1)", &tag); + // bool gotNum = dict.getNum("second(1)", &num); bool exists = dict.exists("second(1)"); diff --git a/src/test/EvioCompStructHandlerTest.cpp b/etc/src/test/cpp/EvioCompStructHandlerTest.cpp similarity index 100% rename from src/test/EvioCompStructHandlerTest.cpp rename to etc/src/test/cpp/EvioCompStructHandlerTest.cpp diff --git a/src/test/FileWriteTest.cpp b/etc/src/test/cpp/FileWriteTest.cpp similarity index 100% rename from src/test/FileWriteTest.cpp rename to etc/src/test/cpp/FileWriteTest.cpp diff --git a/src/test/Hipo_Test.cpp b/etc/src/test/cpp/Hipo_Test.cpp similarity index 100% rename from src/test/Hipo_Test.cpp rename to etc/src/test/cpp/Hipo_Test.cpp diff --git a/src/test/IndexArrayLenTest.cpp b/etc/src/test/cpp/IndexArrayLenTest.cpp similarity index 100% rename from src/test/IndexArrayLenTest.cpp rename to etc/src/test/cpp/IndexArrayLenTest.cpp diff --git a/src/test/README.md b/etc/src/test/cpp/README.md similarity index 100% rename from src/test/README.md rename to etc/src/test/cpp/README.md diff --git a/src/test/ReadSequenceTest.cpp b/etc/src/test/cpp/ReadSequenceTest.cpp similarity index 100% rename from src/test/ReadSequenceTest.cpp rename to etc/src/test/cpp/ReadSequenceTest.cpp diff --git a/src/test/ReadWriteTest.cpp b/etc/src/test/cpp/ReadWriteTest.cpp similarity index 99% rename from src/test/ReadWriteTest.cpp rename to etc/src/test/cpp/ReadWriteTest.cpp index 767d0868b..a96017eaf 100644 --- a/src/test/ReadWriteTest.cpp +++ b/etc/src/test/cpp/ReadWriteTest.cpp @@ -194,7 +194,7 @@ namespace evio { // Do a parsing listener test here auto parser = reader3.getParser(); - // Define a listener to be used with an event parser
IEvioListener listener = new IEvioListener() {
 + // Define a listener to be used with an event parserIEvioListener listener = new IEvioListener() { class myListener : public IEvioListener { public: void gotStructure(std::shared_ptr topStructure, @@ -213,7 +213,7 @@ namespace evio { auto listener = std::make_shared(); - // Define a listener to be used with an event parser
IEvioListener listener = new IEvioListener() {
 + // Define a listener to be used with an event parserIEvioListener listener = new IEvioListener() { class myListener2 : public IEvioListener { public: void gotStructure(std::shared_ptr topStructure, diff --git a/etc/src/test/cpp/ReadWriteV4Test.cpp b/etc/src/test/cpp/ReadWriteV4Test.cpp new file mode 100644 index 000000000..993c3cee3 --- /dev/null +++ b/etc/src/test/cpp/ReadWriteV4Test.cpp @@ -0,0 +1,382 @@ +/** + * Copyright (c) 2024, Jefferson Science Associates + * + * Thomas Jefferson National Accelerator Facility + * Data Acquisition Group + * + * 12000, Jefferson Ave, Newport News, VA 23606 + * Phone : (757)-269-7100 + * + * @date 07/25/2024 + * @author timmer + */ + + + + #include + #include + #include + #include + + + #include "TestBase.h" + + + using namespace std; + + + namespace evio { + + + class ReadWriteTest : public TestBase { + + + public: + + + + std::shared_ptr generateEvioBank(ByteOrder & order, uint16_t tag, uint8_t num) { + // Event, traditional bank of banks + EventBuilder builder(tag, DataType::BANK, num); + auto ev = builder.getEvent(); + + // add a bank of doubles + auto bank1 = EvioBank::getInstance(22, DataType::DOUBLE64, 0); + double dData[1000]; + for (int i = 0; i < 1000; i++) { + dData[i] = i + 1.; + } + builder.appendDoubleData(bank1, dData, 1000); + cout << " generate Evio Bank, bank1 len = " << bank1->getTotalBytes() << endl; + builder.addChild(ev, bank1); + cout << " generate Evio Bank, ev len = " << ev->getTotalBytes() << endl; + + return static_cast>(ev); + } + + + + void writeFile(string finalFilename, uint16_t tag, uint8_t num) { + + std::cout << std::endl; + std::cout << "--------------------------------------------" << std::endl; + std::cout << "---------- Write to file --------------" << std::endl; + std::cout << "--------------------------------------------" << std::endl; + + ByteOrder outputOrder = ByteOrder::ENDIAN_LITTLE; + + // Create a "first event" + uint32_t firstEventData[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + uint32_t firstEventDataLen = 10; + EventBuilder builder(1, DataType::UINT32, 2); + std::shared_ptr firstEvent = builder.getEvent(); + builder.appendUIntData(firstEvent, firstEventData, firstEventDataLen); + //auto firstBank = static_cast>(firstEvent); + + std::string directory; + std::string runType; + + // Create files + EventWriterV4 writer(finalFilename, directory, runType, 1, 0, + EventWriterV4::DEFAULT_BLOCK_SIZE, + EventWriterV4::DEFAULT_BLOCK_COUNT, + outputOrder, dictionary, + true, false, + firstEvent); + + + // EventWriterV4(std::string & baseName, + // const std::string & directory, const std::string & runType, + // uint32_t runNumber = 1, uint64_t split = 0, + // uint32_t maxBlockSize = DEFAULT_BLOCK_SIZE, + // uint32_t maxEventCount = DEFAULT_BLOCK_COUNT, + // const ByteOrder & byteOrder = ByteOrder::nativeOrder(), + // const std::string & xmlDictionary = "", + // bool overWriteOK = true, bool append = false, + // std::shared_ptr firstEvent = nullptr, + // uint32_t streamId = 0, uint32_t splitNumber = 0, + // uint32_t splitIncrement = 1, uint32_t streamCount = 1, + // uint32_t bufferSize = DEFAULT_BUFFER_SIZE, + // std::bitset<24> *bitInfo = nullptr); + + + // Create an event with lots of stuff in it + auto evioDataBuf = createCompactEventBuffer(tag, num, outputOrder); + + // Create node from this buffer + std::shared_ptr node = EvioNode::extractEventNode(evioDataBuf,0,0,0); + + // Create EvioBank + std::shared_ptr bank = generateEvioBank(outputOrder, tag, num); + + // write as buffer + writer.writeEvent(evioDataBuf, false); + cout << " Wrote evio buffer, len = " << evioDataBuf->limit() << endl; + + // write as node + writer.writeEvent(node, false); + cout << " Wrote evio node, total bytes = " << node->getTotalBytes() << endl; + + // write as EvioBank + writer.writeEvent(bank); + cout << " Wrote evio bank, total bytes = " << bank->getTotalBytes() << endl; + cout << " Wrote evio bank, header len in bytes = " << 4*(bank->getHeader()->getLength() + 1) << endl; + + writer.close(); + cout << "Finished writing file " << finalFilename << " now read it" << endl; + + //Util::printBytes(finalFilename, 0, 2000, "File bytes"); + } + + + void readFile(string finalFilename) { + + std::cout << std::endl; + std::cout << "--------------------------------------------" << std::endl; + std::cout << "---------- Read from file --------------" << std::endl; + std::cout << "--------------------------------------------" << std::endl; + + EvioReader reader(finalFilename); + ByteOrder & order = reader.getByteOrder(); + + cout << "Read in file " << finalFilename << " of byte order " << order.getName() << endl; + int32_t evCount = reader.getEventCount(); + cout << "Got " << evCount << " events" << endl; + + string dict = reader.getDictionaryXML(); + if (dict.empty()) { + cout << "\nNo dictionary" << endl; + } + else { + cout << "\nGot dictionary:\n" << dict << endl; + } + + auto pFE = reader.getFirstEvent(); + if (pFE != nullptr) { + cout << "\nGot first Event:\n" << pFE->toString() << endl << endl; + } + else { + cout << "\nNo first event" << endl; + } + + cout << "Print out regular events:" << endl; + + for (int i=0; i < evCount; i++) { + auto ev = reader.getEvent(i+1); + cout << "\nEvent" << (i+1) << ":\n" << ev->toString() << endl; + } + } + + + + void writeAndReadBuffer(uint16_t tag, uint8_t num) { + + std::cout << std::endl; + std::cout << "--------------------------------------------" << std::endl; + std::cout << "---------- Write to buf --------------" << std::endl; + std::cout << "--------------------------------------------" << std::endl; + + ByteOrder order = ByteOrder::ENDIAN_LITTLE; + std::shared_ptr buffer = std::make_shared(200000); + buffer->order(order); + + // Create a "first event" + int firstEventData[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + EventBuilder builder(1, DataType::INT32, 2); + std::shared_ptr firstEv = builder.getEvent(); + try { + builder.setIntData(firstEv, firstEventData, 10); + } + catch (EvioException &e) {/* never happen */} + + bool append = false; + // Allow only 2 events per block to test using multiple blocks + int blockCount = 20000; + + try { + EventWriterV4 writer( + buffer, + EventWriterV4::DEFAULT_BLOCK_SIZE, + blockCount, + dictionary, nullptr, 0, + 1, append, firstEv); + + // Create an event in buffer form with lots of stuff in it + std::shared_ptr evioDataBuf = createCompactEventBuffer(tag, num, order); + // Create same event as EvioEvent object + std::shared_ptr evioEv = createEventBuilderEvent(tag, num); + // Create node from this buffer + std::shared_ptr node = EvioNode::extractEventNode(evioDataBuf, 0, 0, 0); + + + // write as buffer + writer.writeEvent(evioDataBuf, false); + // write as EvioEvent + writer.writeEvent(evioEv, false); + // write as node + writer.writeEvent(node, false); + + writer.close(); + // Get ready-to-read buffer + buffer = writer.getByteBuffer(); + + } + catch (EvioException &e) { + std::cout << "PROBLEM: " << e.what(); + } + + std::shared_ptr copy = ByteBuffer::copyBuffer(buffer); + std::shared_ptr copy2 = ByteBuffer::copyBuffer(buffer); + + + // Reader cannot be used, it only works on evio version 6 files and buffers. + // Reader reader(buffer); + + + + std::cout << "--------------------------------------------" << std::endl; + std::cout << "---------- EvioCompactReader -----------" << std::endl; + std::cout << "--------------------------------------------" << std::endl; + + std::shared_ptr dataBuf0 = nullptr; + + try { + EvioCompactReader reader1(copy); + + uint32_t evCount2 = reader1.getEventCount(); + std::cout << " Got " << evCount2 << " events" << std::endl; + + std::string dict2 = reader1.getDictionaryXML(); + std::cout << " Got dictionary = \n" << dict2 << std::endl; + + // Compact reader does not deal with first events, so skip over it + + std::cout << "\n Print out events (includes first event if evio version 4) :" << std::endl; + + for (uint32_t i = 0; i < evCount2; i++) { + // The "first event" is just the first event in the list (not treated specially) + std::cout << " scanned event #" << (i+1) << " :" << std::endl; + std::shared_ptr compactNode = reader1.getScannedEvent(i + 1); + std::cout << " node ->\n " << compactNode->toString() << std::endl; + + std::shared_ptr dataBuf = std::make_shared(compactNode->getTotalBytes()); + compactNode->getStructureBuffer(dataBuf, true); + + if (i == 0) { + dataBuf0 = dataBuf; + Util::printBytes(dataBuf, dataBuf->position(), dataBuf->remaining(), + " Event #" + std::to_string(i+1)); + } + } + } + catch (EvioException &e) { + std::cout << "PROBLEM: " << e.what(); + } + + + std::cout << std::endl; + std::cout << "--------------------------------------------" << std::endl; + std::cout << "---------- EvioReader --------------" << std::endl; + std::cout << "--------------------------------------------" << std::endl; + + + bool unchanged = true; + size_t index = 0; + std::vector dataVec; + std::vector dataVec0; + + try { + + // TODO: Something WRONG with the parsing!!! + EvioReader reader1(copy2); + + uint32_t evCount2 = reader1.getEventCount(); + std::cout << " Got " << evCount2 << " events" << std::endl; + + std::string dict2 = reader1.getDictionaryXML(); + std::cout << " Got dictionary = \n" << dict2 << std::endl; + + std::cout << "\n Got first event = " << reader1.hasFirstEvent() << std::endl; + + std::cout << "\n Print out events (includes first event if evio version 4) :" << std::endl; + + for (uint32_t i = 0; i < evCount2; i++) { + std::shared_ptr ev = reader1.parseEvent(i + 1); + std::cout << " got & parsed ev " << (i+1) << std::endl; + std::cout << " event ->\n" << ev->toString() << std::endl; + if (i == 0) { + dataVec0 = ev->getRawBytes(); + } + //std::shared_ptr evt = reader1.parseNextEvent(); + } + // This has the same output as above + // int j=0; + // std::shared_ptr evt; + // while ((evt = reader1.parseNextEvent()) != nullptr) { + // std::cout << " got & parsed ev " << (j+1) << std::endl; + // std::cout << " event ->\n" << evt->toString() << std::endl; + // if (j == 0) { + // dataVec0 = evt->getRawBytes(); + // } + // j++; + // } + + std::cout << " Comparing buffer data (lim = " << dataBuf0->limit() << ") with vector data (len = " << dataVec0.size() + << ")" << std::endl; + for (size_t i = 0; i < dataVec0.size(); i++) { + if ((/*data[i+8]*/ dataBuf0->array()[i + 8] != dataVec0[i]) && (i > 3)) { + unchanged = false; + index = i; + std::cout << " Compact reader different than EvioReader at byte #" << index << std::endl; + std::cout << showbase << hex << dataBuf0->array()[i + 8] << " changed to " << + dataVec0[i] << dec << std::endl; + break; + } + } + if (unchanged) { + std::cout << "First data EVENT same whether using EvioCompactReader or EvioReader!" << std::endl; + } + + } + catch (EvioException &e) { + std::cout << "PROBLEM: " << e.what(); + } + + + + } + + + }; + + + } + + + + + int main(int argc, char **argv) { + + + string filename_c = "./evioTest.c.evio"; + string filename_j = "./evioTest.java.evio"; + + evio::ReadWriteTest tester; + + tester.writeFile(filename_c, 1,1); + tester.readFile(filename_c); + + // std::ifstream file(filename_j); + // if (file) { + // tester.readFile(filename_j); + // } + + tester.writeAndReadBuffer(1,1); + + // Buffers ... + cout << endl << endl << "----------------------------------------" << endl << endl; + + return 0; + } + + \ No newline at end of file diff --git a/src/test/RecordSupplyTest.cpp b/etc/src/test/cpp/RecordSupplyTest.cpp similarity index 100% rename from src/test/RecordSupplyTest.cpp rename to etc/src/test/cpp/RecordSupplyTest.cpp diff --git a/src/test/RingBufferTest.cpp b/etc/src/test/cpp/RingBufferTest.cpp similarity index 100% rename from src/test/RingBufferTest.cpp rename to etc/src/test/cpp/RingBufferTest.cpp diff --git a/src/test/SConscript b/etc/src/test/cpp/SConscript similarity index 95% rename from src/test/SConscript rename to etc/src/test/cpp/SConscript index 0c8aacb9e..27dd2cb8a 100644 --- a/src/test/SConscript +++ b/etc/src/test/cpp/SConscript @@ -17,7 +17,7 @@ if platform == 'Darwin': for fileName in ccfileList: compiledFile = env.Program(source = [fileName, ccTestBase], LIBS = ['eviocc'+debugSuffix, 'Disruptor', - 'boost_system', boostThdLib, 'boost_chrono', execLibs], + 'boost_system', boostThdLib, 'boost_chrono', 'boost_filesystem', execLibs], LIBPATH = ['#src/libsrc++/'+archDir, '/usr/lib64', '/usr/lib', '/usr/local/lib', str(disruptorHome)+'/build/Disruptor'], CPPPATH = ['#src/libsrc++', '#src/test', disruptorHome, '/usr/local/include', '/usr/include/boost', '/usr/local/include/boost']) diff --git a/src/test/SequentialReaderTest.cpp b/etc/src/test/cpp/SequentialReaderTest.cpp similarity index 100% rename from src/test/SequentialReaderTest.cpp rename to etc/src/test/cpp/SequentialReaderTest.cpp diff --git a/src/test/SplitTest2.cpp b/etc/src/test/cpp/SplitTest2.cpp similarity index 100% rename from src/test/SplitTest2.cpp rename to etc/src/test/cpp/SplitTest2.cpp diff --git a/src/test/SwapTest.cpp b/etc/src/test/cpp/SwapTest.cpp similarity index 100% rename from src/test/SwapTest.cpp rename to etc/src/test/cpp/SwapTest.cpp diff --git a/src/test/TestBase.cpp b/etc/src/test/cpp/TestBase.cpp similarity index 100% rename from src/test/TestBase.cpp rename to etc/src/test/cpp/TestBase.cpp diff --git a/src/test/TestBase.h b/etc/src/test/cpp/TestBase.h similarity index 100% rename from src/test/TestBase.h rename to etc/src/test/cpp/TestBase.h diff --git a/src/test/Tree_Buf_Composite_Builder_Test.cpp b/etc/src/test/cpp/Tree_Buf_Composite_Builder_Test.cpp similarity index 100% rename from src/test/Tree_Buf_Composite_Builder_Test.cpp rename to etc/src/test/cpp/Tree_Buf_Composite_Builder_Test.cpp diff --git a/src/test/fileTest.cpp b/etc/src/test/cpp/fileTest.cpp similarity index 100% rename from src/test/fileTest.cpp rename to etc/src/test/cpp/fileTest.cpp diff --git a/src/test/hallDEventFiles.cpp b/etc/src/test/cpp/hallDEventFiles.cpp similarity index 100% rename from src/test/hallDEventFiles.cpp rename to etc/src/test/cpp/hallDEventFiles.cpp diff --git a/java/org/jlab/coda/hipo/test/README.md b/etc/src/test/java/hipo/README.md similarity index 100% rename from java/org/jlab/coda/hipo/test/README.md rename to etc/src/test/java/hipo/README.md diff --git a/java/org/jlab/coda/hipo/test/ReadWriteTest.java b/etc/src/test/java/hipo/ReadWriteTest.java similarity index 100% rename from java/org/jlab/coda/hipo/test/ReadWriteTest.java rename to etc/src/test/java/hipo/ReadWriteTest.java diff --git a/java/org/jlab/coda/hipo/test/RecordSupplyTester.java b/etc/src/test/java/hipo/RecordSupplyTester.java similarity index 100% rename from java/org/jlab/coda/hipo/test/RecordSupplyTester.java rename to etc/src/test/java/hipo/RecordSupplyTester.java diff --git a/java/org/jlab/coda/hipo/test/TestBuilding.java b/etc/src/test/java/hipo/TestBuilding.java similarity index 100% rename from java/org/jlab/coda/hipo/test/TestBuilding.java rename to etc/src/test/java/hipo/TestBuilding.java diff --git a/java/org/jlab/coda/hipo/test/TestWriter.java b/etc/src/test/java/hipo/TestWriter.java similarity index 100% rename from java/org/jlab/coda/hipo/test/TestWriter.java rename to etc/src/test/java/hipo/TestWriter.java diff --git a/java/org/jlab/coda/jevio/test/BigFileWrite.java b/etc/src/test/java/jevio/BigFileWrite.java similarity index 100% rename from java/org/jlab/coda/jevio/test/BigFileWrite.java rename to etc/src/test/java/jevio/BigFileWrite.java diff --git a/java/org/jlab/coda/jevio/test/CompactReaderAddSubtractTest.java b/etc/src/test/java/jevio/CompactReaderAddSubtractTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/CompactReaderAddSubtractTest.java rename to etc/src/test/java/jevio/CompactReaderAddSubtractTest.java diff --git a/java/org/jlab/coda/jevio/test/CompactReaderBugTest.java b/etc/src/test/java/jevio/CompactReaderBugTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/CompactReaderBugTest.java rename to etc/src/test/java/jevio/CompactReaderBugTest.java diff --git a/java/org/jlab/coda/jevio/test/CompactStructureHandlerTest.java b/etc/src/test/java/jevio/CompactStructureHandlerTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/CompactStructureHandlerTest.java rename to etc/src/test/java/jevio/CompactStructureHandlerTest.java diff --git a/java/org/jlab/coda/jevio/test/CompositeTester.java b/etc/src/test/java/jevio/CompositeTester.java similarity index 100% rename from java/org/jlab/coda/jevio/test/CompositeTester.java rename to etc/src/test/java/jevio/CompositeTester.java diff --git a/java/org/jlab/coda/jevio/test/CompressionTest.java b/etc/src/test/java/jevio/CompressionTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/CompressionTest.java rename to etc/src/test/java/jevio/CompressionTest.java diff --git a/java/org/jlab/coda/jevio/test/DictTest.java b/etc/src/test/java/jevio/DictTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/DictTest.java rename to etc/src/test/java/jevio/DictTest.java diff --git a/java/org/jlab/coda/jevio/test/ExtractHallDdata.java b/etc/src/test/java/jevio/ExtractHallDdata.java similarity index 100% rename from java/org/jlab/coda/jevio/test/ExtractHallDdata.java rename to etc/src/test/java/jevio/ExtractHallDdata.java diff --git a/java/org/jlab/coda/jevio/test/FileTest.java b/etc/src/test/java/jevio/FileTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/FileTest.java rename to etc/src/test/java/jevio/FileTest.java diff --git a/java/org/jlab/coda/jevio/test/FileTestVer6.java b/etc/src/test/java/jevio/FileTestVer6.java similarity index 100% rename from java/org/jlab/coda/jevio/test/FileTestVer6.java rename to etc/src/test/java/jevio/FileTestVer6.java diff --git a/java/org/jlab/coda/jevio/test/FileWriteTest.java b/etc/src/test/java/jevio/FileWriteTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/FileWriteTest.java rename to etc/src/test/java/jevio/FileWriteTest.java diff --git a/java/org/jlab/coda/jevio/test/FileWritingSpeedTest.java b/etc/src/test/java/jevio/FileWritingSpeedTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/FileWritingSpeedTest.java rename to etc/src/test/java/jevio/FileWritingSpeedTest.java diff --git a/java/org/jlab/coda/jevio/test/FirstEventTest.java b/etc/src/test/java/jevio/FirstEventTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/FirstEventTest.java rename to etc/src/test/java/jevio/FirstEventTest.java diff --git a/java/org/jlab/coda/jevio/test/HipoTest.java b/etc/src/test/java/jevio/HipoTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/HipoTest.java rename to etc/src/test/java/jevio/HipoTest.java diff --git a/java/org/jlab/coda/jevio/test/README.md b/etc/src/test/java/jevio/README.md similarity index 100% rename from java/org/jlab/coda/jevio/test/README.md rename to etc/src/test/java/jevio/README.md diff --git a/java/org/jlab/coda/jevio/test/ReadAggOutput.java b/etc/src/test/java/jevio/ReadAggOutput.java similarity index 100% rename from java/org/jlab/coda/jevio/test/ReadAggOutput.java rename to etc/src/test/java/jevio/ReadAggOutput.java diff --git a/java/org/jlab/coda/jevio/test/ReadSequenceTest.java b/etc/src/test/java/jevio/ReadSequenceTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/ReadSequenceTest.java rename to etc/src/test/java/jevio/ReadSequenceTest.java diff --git a/java/org/jlab/coda/jevio/test/ReadWriteTest.java b/etc/src/test/java/jevio/ReadWriteTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/ReadWriteTest.java rename to etc/src/test/java/jevio/ReadWriteTest.java diff --git a/java/org/jlab/coda/jevio/test/ReadWriteV4Test.java b/etc/src/test/java/jevio/ReadWriteV4Test.java similarity index 100% rename from java/org/jlab/coda/jevio/test/ReadWriteV4Test.java rename to etc/src/test/java/jevio/ReadWriteV4Test.java diff --git a/java/org/jlab/coda/jevio/test/RecordSupplyTest.java b/etc/src/test/java/jevio/RecordSupplyTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/RecordSupplyTest.java rename to etc/src/test/java/jevio/RecordSupplyTest.java diff --git a/java/org/jlab/coda/jevio/test/SequentialReaderTest.java b/etc/src/test/java/jevio/SequentialReaderTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/SequentialReaderTest.java rename to etc/src/test/java/jevio/SequentialReaderTest.java diff --git a/java/org/jlab/coda/jevio/test/SplitTest.java b/etc/src/test/java/jevio/SplitTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/SplitTest.java rename to etc/src/test/java/jevio/SplitTest.java diff --git a/java/org/jlab/coda/jevio/test/SwapTest.java b/etc/src/test/java/jevio/SwapTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/SwapTest.java rename to etc/src/test/java/jevio/SwapTest.java diff --git a/java/org/jlab/coda/jevio/test/TestBase.java b/etc/src/test/java/jevio/TestBase.java similarity index 100% rename from java/org/jlab/coda/jevio/test/TestBase.java rename to etc/src/test/java/jevio/TestBase.java diff --git a/java/org/jlab/coda/jevio/test/TreeBufTest.java b/etc/src/test/java/jevio/TreeBufTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/TreeBufTest.java rename to etc/src/test/java/jevio/TreeBufTest.java diff --git a/java/org/jlab/coda/jevio/test/XmlTest.java b/etc/src/test/java/jevio/XmlTest.java similarity index 100% rename from java/org/jlab/coda/jevio/test/XmlTest.java rename to etc/src/test/java/jevio/XmlTest.java diff --git a/java/jars/README.md b/java/jars/README.md new file mode 100644 index 000000000..b45124075 --- /dev/null +++ b/java/jars/README.md @@ -0,0 +1,5 @@ +# **Java JARs Folder** + +This folder contains any jar file dependencies required by the EVIO Java library that are not retrieved directly from the Maven repository. At present, only the disruptor library needs to be added from here. + +Note that the disruptor 4.0.0 jar file included comes from a [JeffersonLab fork](https://github.com/JeffersonLab/disruptor) of the original lmax Java library. This forked repository adds an additional function (SpinCountBackoffWaitStrategy) that is required in the evio Java Library. \ No newline at end of file diff --git a/java/jars/disruptor-4.0.0.jar b/java/jars/disruptor-4.0.0.jar new file mode 100644 index 000000000..c0fcac29a Binary files /dev/null and b/java/jars/disruptor-4.0.0.jar differ diff --git a/java/org/jlab/coda/jevio/manifest.mf b/java/org/jlab/coda/jevio/manifest.mf deleted file mode 100644 index f88bf6d05..000000000 --- a/java/org/jlab/coda/jevio/manifest.mf +++ /dev/null @@ -1,5 +0,0 @@ -Manifest-Version: 1.0 -Created-By: "Dave Heddle" -Specification-Title: jevio -Sealed: true -Main-Class: org.jlab.coda.jevio.test.HallDEventFiles diff --git a/pom.xml b/pom.xml index e8aa0494a..f8956ceb2 100644 --- a/pom.xml +++ b/pom.xml @@ -1,215 +1,168 @@ - - - - - 4.0.0 - - - jevio - - - Java EVIO Library - - - Java EVIO Library (JEVIO) - - - org.jlab.coda - - - 6.0-SNAPSHOT - - - - - 1.8 - 1.8 - - - - 1.8.0 - 3.4.3 - 1.0 - - - - - jar - - - - net.jpountz.lz4 - lz4-java - ${lz4.version} - - + + 4.0.0 + org.jlab.coda + jevio + 6.0.0 + Jefferson Lab EVIO Java Library (JEVIO) + Java library for EVIO (Event Input/Output) + jar + + + + + + + + 15 + ${java.version} + ${java.version} + + 1.8.0 + 4.0.0 + + UTF-8 + ${pom.basedir}/java/jars + + + + + + + local-disruptor + file://${project.basedir}/java/jars + true + + + + + + + org.lz4 + lz4-java + ${lz4.version} + + + + com.lmax + disruptor + ${disruptor.version} + + + + org.junit.jupiter + junit-jupiter + 5.9.3 + test + + + + + + + + + + + org.apache.maven.plugins + maven-install-plugin + 3.1.4 + + + install-local-disruptor + initialize + + install-file + + + ${project.basedir}/java/jars/disruptor-${disruptor.version}.jar com.lmax disruptor ${disruptor.version} - - - com.aha.compression - AHACompressionAPI - ${aha.version} - - - org.junit.jupiter - junit-jupiter - RELEASE - test - - - org.junit.jupiter - junit-jupiter - RELEASE - test - - - - - - - ./java/ - - - - - - - org.apache.maven.wagon - wagon-ssh-external - 2.8 - - - - - - - - - org.apache.maven.plugins - maven-install-plugin - 2.5.2 - - - aha - clean - install-file - - java/jars/AHACompressionAPI.jar - com.aha.compression - AHACompressionAPI - ${aha.version} - jar - - - - disruptor - clean - install-file - - java/jars/disruptor-${disruptor.version}.jar - com.lmax - disruptor - ${disruptor.version} - jar - - - - lz4 - clean - install-file - - java/jars/lz4-java-${lz4.version}.jar - net.jpountz.lz4 - lz4-java - ${lz4.version} - jar - - - - - - - maven-assembly-plugin - 2.6 - - - - org.jlab.coda.hipo.Evio6Converter - - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - copy-installed - install - - copy - - - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - - - - - - - - - - - - ssh-clasweb - scpexe://clas12@jlabl5/group/clas/www/clasweb/html/clas12maven - - + jar + true + + + + + + + + + + + maven-assembly-plugin + 3.6.0 + + + + + jar-with-dependencies + + + jar-with-dependencies + jar + false + + + runtime + true + + + system + true + false + / + + + + + + + make-assembly + package + + single + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M7 + + + + **/**.java + + + true + + + + + + + diff --git a/src/README.md b/src/README.md new file mode 100644 index 000000000..f99503b21 --- /dev/null +++ b/src/README.md @@ -0,0 +1,9 @@ +# **Source Code for EVIO** + +| Folder | Contents +|------------|---------------------------------------------------------------------- +| [libsrc](libsrc) | EVIO C library | +| [libsrc++](libsrc++) | EVIO C++ library | +| [main/java](main/java) | EVIO Java Library | +| [test](test) | For internal testing of Java, C, C++ libraries. | + diff --git a/src/libsrc++/BaseStructure.cpp b/src/libsrc++/BaseStructure.cpp index 7f0ca5d2c..424b1e7a7 100644 --- a/src/libsrc++/BaseStructure.cpp +++ b/src/libsrc++/BaseStructure.cpp @@ -367,8 +367,8 @@ namespace evio { * * @param newParent this node's new parent. */ - void BaseStructure::setParent(std::weak_ptr newParent) {parent = std::move(newParent);} + void BaseStructure::setParent(std::weak_ptr newParent) {parent = std::move(newParent);} /** * Removes newChild from its present parent (if it has a diff --git a/src/libsrc++/EventBuilder.cpp b/src/libsrc++/EventBuilder.cpp index 17b4abf30..43e21550e 100644 --- a/src/libsrc++/EventBuilder.cpp +++ b/src/libsrc++/EventBuilder.cpp @@ -142,7 +142,7 @@ namespace evio { } child->removeFromParent(); - child->setParent(std::weak_ptr()); + child->setParent(std::weak_ptr()); setAllHeaderLengths(); } diff --git a/src/libsrc++/EvioCompactReaderV6.cpp b/src/libsrc++/EvioCompactReaderV6.cpp index 96d56aada..129dc444c 100644 --- a/src/libsrc++/EvioCompactReaderV6.cpp +++ b/src/libsrc++/EvioCompactReaderV6.cpp @@ -22,10 +22,16 @@ namespace evio { */ EvioCompactReaderV6::EvioCompactReaderV6(std::string const & fileName) { if (fileName.empty()) { - throw EvioException("Buffer arg is emptyl"); + throw EvioException("Buffer arg is empty"); } path = fileName; reader.open(fileName); + + if(reader.getFileHeader().getFileId() == reader.getFileHeader().HIPO_FILE_UNIQUE_WORD) { + throw EvioException("\n File is in HIPO format, not evio. Use HIPO reader instead (https://github.com/gavalian/hipo)"); + } + + } diff --git a/src/libsrc++/EvioReaderV6.cpp b/src/libsrc++/EvioReaderV6.cpp index 429802486..f608729c0 100644 --- a/src/libsrc++/EvioReaderV6.cpp +++ b/src/libsrc++/EvioReaderV6.cpp @@ -37,7 +37,11 @@ namespace evio { synchronized = synced; reader = std::make_shared(path, checkSeq, forceScan); parser = std::make_shared(); - } + + if (reader->getFileHeader().getFileId() == reader->getFileHeader().HIPO_FILE_UNIQUE_WORD) { + throw EvioException("\n File is in HIPO format, not evio. Use HIPO reader instead (https://github.com/gavalian/hipo)"); + } + } /** diff --git a/src/libsrc++/FileHeader.h b/src/libsrc++/FileHeader.h index 52630e5ee..9602fc933 100644 --- a/src/libsrc++/FileHeader.h +++ b/src/libsrc++/FileHeader.h @@ -96,9 +96,9 @@ namespace evio { public: /** First word in every HIPO file for identification purposes. */ - static const uint32_t HIPO_FILE_UNIQUE_WORD = 0x4F504948; // 0x4849504F = HIPO + static const uint32_t HIPO_FILE_UNIQUE_WORD = 0x4F504948; // 0x4849504F for HIPO (= "OPIH" in ascii) /** First word in every Evio file for identification purposes. */ - static const uint32_t EVIO_FILE_UNIQUE_WORD = 0x4556494F; // = EVIO + static const uint32_t EVIO_FILE_UNIQUE_WORD = 0x4556494F; // = "EVIO" in ascii /** Number of 32-bit words in a normal sized header. */ static const uint32_t HEADER_SIZE_WORDS = 14; /** Number of bytes in a normal sized header. */ diff --git a/java/org/jlab/coda/hipo/CompressionType.java b/src/main/java/org/jlab/coda/hipo/CompressionType.java similarity index 100% rename from java/org/jlab/coda/hipo/CompressionType.java rename to src/main/java/org/jlab/coda/hipo/CompressionType.java diff --git a/java/org/jlab/coda/hipo/Compressor.java b/src/main/java/org/jlab/coda/hipo/Compressor.java similarity index 100% rename from java/org/jlab/coda/hipo/Compressor.java rename to src/main/java/org/jlab/coda/hipo/Compressor.java diff --git a/java/org/jlab/coda/hipo/DataUtils.java b/src/main/java/org/jlab/coda/hipo/DataUtils.java similarity index 100% rename from java/org/jlab/coda/hipo/DataUtils.java rename to src/main/java/org/jlab/coda/hipo/DataUtils.java diff --git a/java/org/jlab/coda/hipo/Evio6Converter.java b/src/main/java/org/jlab/coda/hipo/Evio6Converter.java similarity index 100% rename from java/org/jlab/coda/hipo/Evio6Converter.java rename to src/main/java/org/jlab/coda/hipo/Evio6Converter.java diff --git a/java/org/jlab/coda/hipo/FileEventIndex.java b/src/main/java/org/jlab/coda/hipo/FileEventIndex.java similarity index 100% rename from java/org/jlab/coda/hipo/FileEventIndex.java rename to src/main/java/org/jlab/coda/hipo/FileEventIndex.java diff --git a/java/org/jlab/coda/hipo/FileHeader.java b/src/main/java/org/jlab/coda/hipo/FileHeader.java similarity index 58% rename from java/org/jlab/coda/hipo/FileHeader.java rename to src/main/java/org/jlab/coda/hipo/FileHeader.java index fb1f65d2d..693cbc54a 100644 --- a/java/org/jlab/coda/hipo/FileHeader.java +++ b/src/main/java/org/jlab/coda/hipo/FileHeader.java @@ -12,7 +12,8 @@ /** * This class represents the file header of an evio or hipo format file. * - * 

+ * 
+ * 
  *
  * FILE HEADER STRUCTURE ( 56 bytes, 14 integers (32 bit) )
  *
@@ -63,7 +64,8 @@
  *                                 5 = HIPO file
  *                                 6 = HIPO extended file
  *
- * 

+ * 
+ *
* * @version 6.0 * @since 6.0 9/6/17 @@ -73,184 +75,195 @@ public class FileHeader implements Cloneable { /** Array to help find number of bytes to pad data. */ - private final static int[] padValue = {0,3,2,1}; + private final static int[] padValue = { 0, 3, 2, 1 }; /** First word in every HIPO file for identification purposes. */ - public final static int HIPO_FILE_UNIQUE_WORD = 0x4F504948; // 0x4849504F = HIPO + public final static int HIPO_FILE_UNIQUE_WORD = 0x4F504948; // 0x4849504F = HIPO /** First word in every Evio file for identification purposes. */ - public final static int EVIO_FILE_UNIQUE_WORD = 0x4556494F; // = EVIO + public final static int EVIO_FILE_UNIQUE_WORD = 0x4556494F; // = EVIO /** Number of 32-bit words in a normal sized header. */ - public final static int HEADER_SIZE_WORDS = 14; + public final static int HEADER_SIZE_WORDS = 14; /** Number of bytes in a normal sized header. */ - public final static int HEADER_SIZE_BYTES = 56; + public final static int HEADER_SIZE_BYTES = 56; /** Magic number used to track endianness. */ - public final static int HEADER_MAGIC = 0xc0da0100; + public final static int HEADER_MAGIC = 0xc0da0100; // Byte offset to header words /** Byte offset from beginning of header to the file id. */ - public final static int FILE_ID_OFFSET = 0; + public final static int FILE_ID_OFFSET = 0; /** Byte offset from beginning of header to the file number. */ - public final static int FILE_NUMBER_OFFSET = 4; + public final static int FILE_NUMBER_OFFSET = 4; /** Byte offset from beginning of header to the header length. */ - public final static int HEADER_LENGTH_OFFSET = 8; + public final static int HEADER_LENGTH_OFFSET = 8; /** Byte offset from beginning of header to the record count. */ - public final static int RECORD_COUNT_OFFSET = 12; + public final static int RECORD_COUNT_OFFSET = 12; /** Byte offset from beginning of header to the index array length. */ - public final static int INDEX_ARRAY_OFFSET = 16; + public final static int INDEX_ARRAY_OFFSET = 16; /** Byte offset from beginning of header to bit info word. */ - public final static int BIT_INFO_OFFSET = 20; + public final static int BIT_INFO_OFFSET = 20; /** Byte offset from beginning of header to the user header length. */ - public final static int USER_LENGTH_OFFSET = 24; + public final static int USER_LENGTH_OFFSET = 24; /** Byte offset from beginning of header to the record length. */ - public final static int MAGIC_OFFSET = 28; + public final static int MAGIC_OFFSET = 28; /** Byte offset from beginning of header to the user register #1. */ - public final static int REGISTER1_OFFSET = 32; + public final static int REGISTER1_OFFSET = 32; /** Byte offset from beginning of header to write trailer position. */ - public final static int TRAILER_POSITION_OFFSET = 40; + public final static int TRAILER_POSITION_OFFSET = 40; /** Byte offset from beginning of header to the user integer #1. */ - public final static int INT1_OFFSET = 48; + public final static int INT1_OFFSET = 48; /** Byte offset from beginning of header to the user integer #2. */ - public final static int INT2_OFFSET = 52; + public final static int INT2_OFFSET = 52; // Bits in bit info word /** 8th bit set in bitInfo word in file header means contains dictionary. */ - final static int DICTIONARY_BIT = 0x100; - /** 9th bit set in bitInfo word in file header means every split file has same first event. */ - final static int FIRST_EVENT_BIT = 0x200; - /** 10th bit set in bitInfo word in file header means file trailer with index array exists. */ - final static int TRAILER_WITH_INDEX_BIT = 0x400; - - + final static int DICTIONARY_BIT = 0x100; + /** + * 9th bit set in bitInfo word in file header means every split file has same + * first event. + */ + final static int FIRST_EVENT_BIT = 0x200; + /** + * 10th bit set in bitInfo word in file header means file trailer with index + * array exists. + */ + final static int TRAILER_WITH_INDEX_BIT = 0x400; - /** File id for file identification purposes. Defaults to HIPO file. 1st word. */ - private int fileId = HIPO_FILE_UNIQUE_WORD; + /** + * File id for file identification purposes. Defaults to HIPO file. 1st word. + */ + private int fileId = HIPO_FILE_UNIQUE_WORD; /** File number or split file number, starting at 1. 2nd word. */ - private int fileNumber = 1; + private int fileNumber = 1; /** User-defined 64-bit register. 9th and 10th words. */ private long userRegister; /** Position of trailing header from start of file in bytes. 11th word. */ private long trailerPosition; /** First user-defined integer in file header. 13th word. */ - private int userIntFirst; + private int userIntFirst; /** Second user-defined integer in file header. 14th word. */ - private int userIntSecond; + private int userIntSecond; /** Position of this header in a file. */ private long position; /** Event or record count. 4th word. */ - private int entries; + private int entries; /** BitInfo & version. 6th word. */ - private int bitInfo = -1; + private int bitInfo = -1; /** Length of this header (bytes). */ - private int headerLength = HEADER_SIZE_BYTES; + private int headerLength = HEADER_SIZE_BYTES; /** Length of this header (words). 3rd word. */ - private int headerLengthWords = HEADER_SIZE_WORDS; + private int headerLengthWords = HEADER_SIZE_WORDS; /** Length of user-defined header (bytes). 7th word. */ - private int userHeaderLength; + private int userHeaderLength; /** Length of user-defined header when padded (words). */ - private int userHeaderLengthWords; + private int userHeaderLengthWords; /** Length of index array (bytes). 5th word. */ - private int indexLength; + private int indexLength; - /** Final, total length of header + index + user header (bytes) + padding. - * Not stored in any word. */ - private int totalLength = HEADER_SIZE_BYTES; + /** + * Final, total length of header + index + user header (bytes) + padding. + * Not stored in any word. + */ + private int totalLength = HEADER_SIZE_BYTES; - /** Evio format version number. It is 6 when being written, else - * the version of file/buffer being read. Lowest byte of 6th word. */ - private int headerVersion = 6; + /** + * Evio format version number. It is 6 when being written, else + * the version of file/buffer being read. Lowest byte of 6th word. + */ + private int headerVersion = 6; /** Magic number for tracking endianness. 8th word. */ - private int headerMagicWord = HEADER_MAGIC; + private int headerMagicWord = HEADER_MAGIC; - /** Number of bytes required to bring uncompressed - * user header to 4-byte boundary. Stored in 6th word. - * Updated automatically when lengths are set. */ - private int userHeaderLengthPadding; + /** + * Number of bytes required to bring uncompressed + * user header to 4-byte boundary. Stored in 6th word. + * Updated automatically when lengths are set. + */ + private int userHeaderLengthPadding; /** Byte order of file. */ - private ByteOrder byteOrder = ByteOrder.LITTLE_ENDIAN; + private ByteOrder byteOrder = ByteOrder.LITTLE_ENDIAN; /** Type of header this is. Normal HIPO record by default. */ private HeaderType headerType = HeaderType.HIPO_FILE; - - /** Default, no-arg constructor. */ - public FileHeader() {bitInfoInit();} + public FileHeader() { + bitInfoInit(); + } /** * Constructor which sets the type of header this is and file id. + * * @param isEvio if true, file has EVIO file id and header type, else is HIPO. */ public FileHeader(boolean isEvio) { if (isEvio) { fileId = EVIO_FILE_UNIQUE_WORD; headerType = HeaderType.EVIO_FILE; - } - else { + } else { fileId = HIPO_FILE_UNIQUE_WORD; headerType = HeaderType.HIPO_FILE; } bitInfoInit(); } - - /*** {@inheritDoc} */ + /*** {@inheritDoc} */ public Object clone() { try { // Don't do anything special as all are primitive types or enum or ByteOrder return super.clone(); - } - catch (CloneNotSupportedException e) { + } catch (CloneNotSupportedException e) { return null; } } - /** * Copy the contents of the arg into this object. + * * @param head object to copy */ public void copy(FileHeader head) { - if (head == null) return; - - fileId = head.fileId; - fileNumber = head.fileNumber; - userRegister = head.userRegister; - trailerPosition = head.trailerPosition; - userIntFirst = head.userIntFirst; - userIntSecond = head.userIntSecond; - position = head.position; - totalLength = head.totalLength; - headerType = head.headerType; - entries = head.entries; - bitInfo = head.bitInfo; - headerLength = head.headerLength; - headerLengthWords = head.headerLengthWords; - userHeaderLength = head.userHeaderLength; - userHeaderLengthWords = head.userHeaderLengthWords; - indexLength = head.indexLength; - headerMagicWord = head.headerMagicWord; + if (head == null) + return; + + fileId = head.fileId; + fileNumber = head.fileNumber; + userRegister = head.userRegister; + trailerPosition = head.trailerPosition; + userIntFirst = head.userIntFirst; + userIntSecond = head.userIntSecond; + position = head.position; + totalLength = head.totalLength; + headerType = head.headerType; + entries = head.entries; + bitInfo = head.bitInfo; + headerLength = head.headerLength; + headerLengthWords = head.headerLengthWords; + userHeaderLength = head.userHeaderLength; + userHeaderLengthWords = head.userHeaderLengthWords; + indexLength = head.indexLength; + headerMagicWord = head.headerMagicWord; userHeaderLengthPadding = head.userHeaderLengthPadding; - headerVersion = head.headerVersion; + headerVersion = head.headerVersion; } /** Reset most internal variables (not file id and header type). */ - public void reset(){ + public void reset() { // Do NOT reset fileId which is only set in constructor. // Do NOT reset header type. fileNumber = 1; userRegister = 0L; trailerPosition = 0L; - userIntFirst = 0; + userIntFirst = 0; userIntSecond = 0; position = 0L; entries = 0; bitInfoInit(); - totalLength = HEADER_SIZE_BYTES; + totalLength = HEADER_SIZE_BYTES; headerLength = HEADER_SIZE_BYTES; headerLengthWords = HEADER_SIZE_WORDS; userHeaderLength = 0; @@ -262,133 +275,194 @@ public void reset(){ /** * Returns length padded to 4-byte boundary for given length in bytes. + * * @param length length in bytes. * @return length in bytes padded to 4-byte boundary. */ - private static int getWords(int length){ - int words = length/4; - if (getPadding(length) > 0) words++; + private static int getWords(int length) { + int words = length / 4; + if (getPadding(length) > 0) + words++; return words; } /** - * Returns number of bytes needed to pad to 4-byte boundary for the given length. + * Returns number of bytes needed to pad to 4-byte boundary for the given + * length. + * * @param length length in bytes. * @return number of bytes needed to pad to 4-byte boundary. */ - private static int getPadding(int length) {return padValue[length%4];} + private static int getPadding(int length) { + return padValue[length % 4]; + } // Getters /** * Get the byte order of the file this header was read from. * Defaults to little endian. + * * @return byte order of the file this header was read from. */ - public ByteOrder getByteOrder() {return byteOrder;} + public ByteOrder getByteOrder() { + return byteOrder; + } /** * Get the file number or split number. + * * @return file number. */ - public int getFileNumber() {return fileNumber;} + public int getFileNumber() { + return fileNumber; + } /** * Get the file id. + * * @return file id. */ - public int getFileId() {return fileId;} + public int getFileId() { + return fileId; + } /** * Get the user register value. + * * @return user register value. */ - public long getUserRegister() {return userRegister;} + public long getUserRegister() { + return userRegister; + } /** * Get the trailer's (trailing header's) file position in bytes. + * * @return trailer's (trailing header's) file position in bytes. */ - public long getTrailerPosition() {return trailerPosition;} + public long getTrailerPosition() { + return trailerPosition; + } /** * Get the first user integer value. + * * @return first user integer value. */ - public int getUserIntFirst() {return userIntFirst;} + public int getUserIntFirst() { + return userIntFirst; + } /** * Get the second user integer value. + * * @return second user integer value. */ - public int getUserIntSecond() {return userIntSecond;} + public int getUserIntSecond() { + return userIntSecond; + } /** * Get the position of this record in a file. + * * @return position of this record in a file. */ - public long getPosition() {return position;} + public long getPosition() { + return position; + } /** * Get the number of events or entries in index. + * * @return number of events or entries in index. */ - public int getEntries() {return entries;} + public int getEntries() { + return entries; + } /** * Get the length of the user-defined header in bytes. + * * @return length of the user-defined header in bytes. */ - public int getUserHeaderLength() {return userHeaderLength;} + public int getUserHeaderLength() { + return userHeaderLength; + } /** * Get the length of the user-defined header in words. + * * @return length of the user-defined header in words. */ - public int getUserHeaderLengthWords() {return userHeaderLengthWords;} + public int getUserHeaderLengthWords() { + return userHeaderLengthWords; + } /** * Get the Evio format version number. + * * @return Evio format version number. */ - public int getVersion() {return headerVersion;} + public int getVersion() { + return headerVersion; + } /** * Get the length of the index array in bytes. + * * @return length of the index array in bytes. */ - public int getIndexLength() {return indexLength;} + public int getIndexLength() { + return indexLength; + } /** * Get the length of this header data in bytes. + * * @return length of this header data in bytes. */ - public int getHeaderLength() {return headerLength;} + public int getHeaderLength() { + return headerLength; + } /** * Get the type of header this is. + * * @return type of header this is. */ - public HeaderType getHeaderType() {return headerType;} + public HeaderType getHeaderType() { + return headerType; + } /** - * Get the total length of header + index + user header (including padding) in bytes. + * Get the total length of header + index + user header (including padding) in + * bytes. * Never compressed. - * @return total length of header + index + user header (including padding) in bytes. + * + * @return total length of header + index + user header (including padding) in + * bytes. */ - public int getLength() {return totalLength;} + public int getLength() { + return totalLength; + } /** - * Get the user header's padding - the number of bytes required to bring uncompressed + * Get the user header's padding - the number of bytes required to bring + * uncompressed * user header to 4-byte boundary. - * @return user header's padding + * + * @return user header's padding */ - public int getUserHeaderLengthPadding() {return userHeaderLengthPadding;} + public int getUserHeaderLengthPadding() { + return userHeaderLengthPadding; + } // Bit info methods /** * Decodes the bit-info word into version, padding and header type. + * * @param word bit-info word. */ private void decodeBitInfoWord(int word) { @@ -399,7 +473,7 @@ private void decodeBitInfoWord(int word) { headerVersion = (word & 0xff); // Header type - headerType = HeaderType.getHeaderType(word >>> 28); + headerType = HeaderType.getHeaderType(word >>> 28); if (headerType == null) { headerType = HeaderType.EVIO_RECORD; } @@ -412,15 +486,19 @@ private void bitInfoInit() { /** * Get the bit info word. + * * @return bit info word. */ - public int getBitInfoWord() {return bitInfo;} + public int getBitInfoWord() { + return bitInfo; + } /** * Set the bit info word and related values. - * @param word bit info word. + * + * @param word bit info word. */ - void setBitInfoWord(int word) { + void setBitInfoWord(int word) { bitInfo = word; decodeBitInfoWord(word); } @@ -428,34 +506,43 @@ void setBitInfoWord(int word) { /** * Set the bit info word for a file header. * Retains current header type, user header length padding, and version. - * @param haveFirst true if file has first event. - * @param haveDictionary true if file has dictionary in user header. - * @param haveTrailerWithIndex true if file has trailer with record length index. + * + * @param haveFirst true if file has first event. + * @param haveDictionary true if file has dictionary in user header. + * @param haveTrailerWithIndex true if file has trailer with record length + * index. * @return new bit info word. */ public int setBitInfo(boolean haveFirst, - boolean haveDictionary, - boolean haveTrailerWithIndex) { + boolean haveDictionary, + boolean haveTrailerWithIndex) { bitInfo = (headerType.getValue() << 28) | - (userHeaderLengthPadding << 20) | - (headerVersion & 0xFF); + (userHeaderLengthPadding << 20) | + (headerVersion & 0xFF); - if (haveFirst) bitInfo |= FIRST_EVENT_BIT; - if (haveDictionary) bitInfo |= DICTIONARY_BIT; - if (haveTrailerWithIndex) bitInfo |= TRAILER_WITH_INDEX_BIT; + if (haveFirst) + bitInfo |= FIRST_EVENT_BIT; + if (haveDictionary) + bitInfo |= DICTIONARY_BIT; + if (haveTrailerWithIndex) + bitInfo |= TRAILER_WITH_INDEX_BIT; return bitInfo; } /** * Get the file's type. + * * @return the file's HeaderType. */ - public HeaderType getFileType() {return headerType;} + public HeaderType getFileType() { + return headerType; + } /** * Get the file's type from the bit info word. + * * @param word bit-info word. * @return the file's HeaderType. */ @@ -472,15 +559,15 @@ public static HeaderType getFileType(int word) { /** * Set the bit which says file has a first event. - * @param hasFirst true if file has a first event. + * + * @param hasFirst true if file has a first event. * @return new bitInfo word. */ public int hasFirstEvent(boolean hasFirst) { if (hasFirst) { // set bit bitInfo |= FIRST_EVENT_BIT; - } - else { + } else { // clear bit bitInfo &= ~FIRST_EVENT_BIT; } @@ -490,28 +577,35 @@ public int hasFirstEvent(boolean hasFirst) { /** * Does this header have a first event in the file header? + * * @return true if header has a first event in the file header, else false. */ - public boolean hasFirstEvent() {return ((bitInfo & FIRST_EVENT_BIT) != 0);} + public boolean hasFirstEvent() { + return ((bitInfo & FIRST_EVENT_BIT) != 0); + } /** - * Does this bitInfo arg indicate the existence of a first event in the file header? + * Does this bitInfo arg indicate the existence of a first event in the file + * header? + * * @param bitInfo bitInfo word. * @return true if header has a first event in the file header, else false. */ - static public boolean hasFirstEvent(int bitInfo) {return ((bitInfo & FIRST_EVENT_BIT) != 0);} + static public boolean hasFirstEvent(int bitInfo) { + return ((bitInfo & FIRST_EVENT_BIT) != 0); + } /** * Set the bit in the file header which says there is a dictionary. - * @param hasDictionary true if file has a dictionary. + * + * @param hasDictionary true if file has a dictionary. * @return new bitInfo word. */ public int hasDictionary(boolean hasDictionary) { if (hasDictionary) { // set bit bitInfo |= DICTIONARY_BIT; - } - else { + } else { // clear bit bitInfo &= ~DICTIONARY_BIT; } @@ -521,28 +615,37 @@ public int hasDictionary(boolean hasDictionary) { /** * Does this header have a dictionary in the file header? + * * @return true if header has a dictionary in the file header, else false. */ - public boolean hasDictionary() {return ((bitInfo & DICTIONARY_BIT) != 0);} + public boolean hasDictionary() { + return ((bitInfo & DICTIONARY_BIT) != 0); + } /** - * Does this bitInfo arg indicate the existence of a dictionary in the file header? + * Does this bitInfo arg indicate the existence of a dictionary in the file + * header? + * * @param bitInfo bitInfo word. * @return true if header has a dictionary in the file header, else false. */ - static public boolean hasDictionary(int bitInfo) {return ((bitInfo & DICTIONARY_BIT) != 0);} + static public boolean hasDictionary(int bitInfo) { + return ((bitInfo & DICTIONARY_BIT) != 0); + } /** - * Set the bit in the file header which says there is a trailer with a record length index. - * @param hasTrailerWithIndex true if file has a trailer with a record length index. + * Set the bit in the file header which says there is a trailer with a record + * length index. + * + * @param hasTrailerWithIndex true if file has a trailer with a record length + * index. * @return new bitInfo word. */ public int hasTrailerWithIndex(boolean hasTrailerWithIndex) { if (hasTrailerWithIndex) { // set bit bitInfo |= TRAILER_WITH_INDEX_BIT; - } - else { + } else { // clear bit bitInfo &= ~TRAILER_WITH_INDEX_BIT; } @@ -552,12 +655,17 @@ public int hasTrailerWithIndex(boolean hasTrailerWithIndex) { /** * Does this file have a trailer with a record length index? + * * @return true if file has a trailer with a record length index, else false. */ - public boolean hasTrailerWithIndex() {return ((bitInfo & TRAILER_WITH_INDEX_BIT) != 0);} + public boolean hasTrailerWithIndex() { + return ((bitInfo & TRAILER_WITH_INDEX_BIT) != 0); + } /** - * Does this bitInfo arg indicate the existence of a trailer with a record length index? + * Does this bitInfo arg indicate the existence of a trailer with a record + * length index? + * * @param bitInfo bitInfo word. * @return true if file has a trailer with a record length index, else false. */ @@ -565,75 +673,107 @@ static public boolean hasTrailerWithIndex(int bitInfo) { return ((bitInfo & TRAILER_WITH_INDEX_BIT) != 0); } - //-------------------------------------------------------------- + // -------------------------------------------------------------- /** * Is this header followed by a user header? + * * @return true if header followed by a user header, else false. */ - public boolean hasUserHeader() {return userHeaderLength > 0;} + public boolean hasUserHeader() { + return userHeaderLength > 0; + } /** - * Does this file have a valid index of record lengths immediately following header? + * Does this file have a valid index of record lengths immediately following + * header? * There should be at least one integer for valid index. * Must have integral number of entries. + * * @return true if file has a valid index, else false. */ - public boolean hasIndex() {return ((indexLength > 3) && (indexLength % 4 == 0));} + public boolean hasIndex() { + return ((indexLength > 3) && (indexLength % 4 == 0)); + } // Setters /** * Set the file number which is the split number starting at 1. - * @param num file number starting at 1. + * + * @param num file number starting at 1. * @return this object. */ - public FileHeader setFileNumber(int num) {fileNumber = num; return this;} + public FileHeader setFileNumber(int num) { + fileNumber = num; + return this; + } /** * Set the first user register. - * @param val first user register. + * + * @param val first user register. * @return this object. */ - public FileHeader setUserRegister(long val) {userRegister = val; return this;} + public FileHeader setUserRegister(long val) { + userRegister = val; + return this; + } /** * Set the first user integer. - * @param val first user integer. + * + * @param val first user integer. * @return this object. */ - public FileHeader setUserIntFirst(int val) {userIntFirst = val; return this;} + public FileHeader setUserIntFirst(int val) { + userIntFirst = val; + return this; + } /** * Set the second user integer. - * @param val second user integer. + * + * @param val second user integer. * @return this object. */ - public FileHeader setUserIntSecond(int val) {userIntSecond = val; return this;} + public FileHeader setUserIntSecond(int val) { + userIntSecond = val; + return this; + } /** * Set this header's type. Normally done in constructor. Limited access. + * * @param type type of header. * @return this object. */ - FileHeader setHeaderType(HeaderType type) {headerType = type; return this;} + FileHeader setHeaderType(HeaderType type) { + headerType = type; + return this; + } /** * Set the position of this record in a file. + * * @param pos position of this record in a file. * @return this object. */ - public FileHeader setPosition(long pos) {position = pos; return this;} + public FileHeader setPosition(long pos) { + position = pos; + return this; + } /** * Set the length of the index array in bytes. * Length is forced to be a multiple of 4! * Sets the total length too. - * @param length length of index array in bytes. + * + * @param length length of index array in bytes. * @return this object. */ public FileHeader setIndexLength(int length) { - indexLength = (length/4)*4; + indexLength = (length / 4) * 4; setLength(headerLength + indexLength + userHeaderLength + userHeaderLengthPadding); return this; } @@ -641,15 +781,20 @@ public FileHeader setIndexLength(int length) { /** * Set the number of record entries. * No compression for other values. + * * @param n number of record entries. * @return this object. */ - public FileHeader setEntries(int n) {entries = n; return this;} + public FileHeader setEntries(int n) { + entries = n; + return this; + } /** * Set the user-defined header's length in bytes and words and the padding. * Sets the total length too. - * @param length user-defined header's length in bytes. + * + * @param length user-defined header's length in bytes. * @return this object. */ public FileHeader setUserHeaderLength(int length) { @@ -662,32 +807,38 @@ public FileHeader setUserHeaderLength(int length) { } /** - * Set the user header's padding - the number of bytes required to bring uncompressed - * user header to 4-byte boundary. Sets the associated value in bitInfo word. - * @param padding user header's padding. - */ - private void setUserHeaderLengthPadding(int padding) { -//System.out.println("setUserHeaderLengthPadding: IN, firstEv bit = " + hasFirstEvent()); - userHeaderLengthPadding = padding; - bitInfo = bitInfo | (userHeaderLengthPadding << 20); -//System.out.println("setUserHeaderLengthPadding: END, firstEv bit = " + hasFirstEvent()); - } + * Set the user header's padding - the number of bytes required to bring + * uncompressed + * user header to 4-byte boundary. Sets the associated value in bitInfo word. + * + * @param padding user header's padding. + */ + private void setUserHeaderLengthPadding(int padding) { + // System.out.println("setUserHeaderLengthPadding: IN, firstEv bit = " + + // hasFirstEvent()); + userHeaderLengthPadding = padding; + bitInfo = bitInfo | (userHeaderLengthPadding << 20); + // System.out.println("setUserHeaderLengthPadding: END, firstEv bit = " + + // hasFirstEvent()); + } /** * Set the this header's length in bytes and words. * Minimum length of 56 (14 words) is enforced. * If length is not a multiple of 4, you're on your own! * Sets the total length too. - * @param length this header's length in bytes. + * + * @param length this header's length in bytes. * @return this object. */ public FileHeader setHeaderLength(int length) { if (length < 56) { length = 56; -//System.out.println("setHeaderLength: attempting to set header length too small (" + hasFirstEvent() + ")"); + // System.out.println("setHeaderLength: attempting to set header length too + // small (" + hasFirstEvent() + ")"); } headerLength = length; - headerLengthWords = length/4; + headerLengthWords = length / 4; setLength(headerLength + indexLength + userHeaderLength + userHeaderLengthPadding); return this; } @@ -696,47 +847,54 @@ public FileHeader setHeaderLength(int length) { * Set the total length in bytes, header + index + user header. * This includes padding and is on a 4-byte boundary. * Never compressed. - * @param length total length in bytes, header + index + user header. + * + * @param length total length in bytes, header + index + user header. * @return this object. */ - public FileHeader setLength(int length) {totalLength = length; return this;} + public FileHeader setLength(int length) { + totalLength = length; + return this; + } - //------------------------------------------------- + // ------------------------------------------------- /** * Writes the file (not record!) header into the given byte buffer. + * * @param buf byte buffer to write file header into. * @param off position in buffer to begin writing. * @throws HipoException if buffer is null or contains too little room. */ - public void writeHeader(ByteBuffer buf, int off) throws HipoException { - - if (buf == null || (buf.limit() - off) < HEADER_SIZE_BYTES) { - throw new HipoException("buf is null or too small"); - } - - buf.putInt ( off, fileId); // 0*4 - buf.putInt ( 4 + off, fileNumber); // 1*4 - buf.putInt ( 8 + off, headerLengthWords); // 2*4 - buf.putInt (12 + off, entries); // 3*4 - buf.putInt (16 + off, indexLength); // 4*4 - buf.putInt (20 + off, getBitInfoWord()); // 5*4 - buf.putInt (24 + off, userHeaderLength); // 6*4 - buf.putInt (28 + off, headerMagicWord); // 7*4 - buf.putLong(32 + off, userRegister); // 8*4 - buf.putLong(40 + off, trailerPosition); // 10*4 - buf.putInt (48 + off, userIntFirst); // 12*4 - buf.putInt (52 + off, userIntSecond); // 13*4 - } - - /** - * Writes the file (not record!) header into the given byte buffer starting at beginning. - * @param buffer byte buffer to write file header into. - * @throws HipoException if buffer is null or contains too little room. - */ - public void writeHeader(ByteBuffer buffer) throws HipoException { - writeHeader(buffer, 0); - } + public void writeHeader(ByteBuffer buf, int off) throws HipoException { + + if (buf == null || (buf.limit() - off) < HEADER_SIZE_BYTES) { + throw new HipoException("buf is null or too small"); + } + + buf.putInt(off, fileId); // 0*4 + buf.putInt(4 + off, fileNumber); // 1*4 + buf.putInt(8 + off, headerLengthWords); // 2*4 + buf.putInt(12 + off, entries); // 3*4 + buf.putInt(16 + off, indexLength); // 4*4 + buf.putInt(20 + off, getBitInfoWord()); // 5*4 + buf.putInt(24 + off, userHeaderLength); // 6*4 + buf.putInt(28 + off, headerMagicWord); // 7*4 + buf.putLong(32 + off, userRegister); // 8*4 + buf.putLong(40 + off, trailerPosition); // 10*4 + buf.putInt(48 + off, userIntFirst); // 12*4 + buf.putInt(52 + off, userIntSecond); // 13*4 + } + + /** + * Writes the file (not record!) header into the given byte buffer starting at + * beginning. + * + * @param buffer byte buffer to write file header into. + * @throws HipoException if buffer is null or contains too little room. + */ + public void writeHeader(ByteBuffer buffer) throws HipoException { + writeHeader(buffer, 0); + } /** * Reads the file header information from a byte buffer and validates @@ -748,7 +906,8 @@ public void writeHeader(ByteBuffer buffer) throws HipoException { * @param offset position of first word to be read. * @throws HipoException if buffer is null, * remaining buffer space (limit - off) is too small, - * data is not in proper format, or version earlier than 6. + * data is not in proper format, or version earlier than + * 6. */ public void readHeader(ByteBuffer buffer, int offset) throws HipoException { @@ -765,20 +924,17 @@ public void readHeader(ByteBuffer buffer, int offset) throws HipoException { if (headerMagicWord == Integer.reverseBytes(HEADER_MAGIC)) { if (buffer.order() == ByteOrder.BIG_ENDIAN) { byteOrder = ByteOrder.LITTLE_ENDIAN; - } - else { + } else { byteOrder = ByteOrder.BIG_ENDIAN; } buffer.order(byteOrder); headerMagicWord = HEADER_MAGIC; - } - else { + } else { // ERROR condition, bad magic word throw new HipoException("buffer not in evio/hipo format? magic int = 0x" + - Integer.toHexString(headerMagicWord)); + Integer.toHexString(headerMagicWord)); } - } - else { + } else { byteOrder = buffer.order(); } @@ -789,28 +945,28 @@ public void readHeader(ByteBuffer buffer, int offset) throws HipoException { throw new HipoException("evio format < 6, = " + headerVersion); } - fileId = buffer.getInt(FILE_ID_OFFSET + offset); - fileNumber = buffer.getInt(FILE_NUMBER_OFFSET + offset); + fileId = buffer.getInt(FILE_ID_OFFSET + offset); + fileNumber = buffer.getInt(FILE_NUMBER_OFFSET + offset); headerLengthWords = buffer.getInt(HEADER_LENGTH_OFFSET + offset); - setHeaderLength(4*headerLengthWords); -//System.out.println(" fileHeader: header len = " + headerLength); - entries = buffer.getInt(RECORD_COUNT_OFFSET + offset); + setHeaderLength(4 * headerLengthWords); + // System.out.println(" fileHeader: header len = " + headerLength); + entries = buffer.getInt(RECORD_COUNT_OFFSET + offset); - indexLength = buffer.getInt(INDEX_ARRAY_OFFSET + offset); -//System.out.println(" fileHeader: index len = " + indexLength); + indexLength = buffer.getInt(INDEX_ARRAY_OFFSET + offset); + // System.out.println(" fileHeader: index len = " + indexLength); setIndexLength(indexLength); - userHeaderLength = buffer.getInt(USER_LENGTH_OFFSET + offset); + userHeaderLength = buffer.getInt(USER_LENGTH_OFFSET + offset); setUserHeaderLength(userHeaderLength); -//System.out.println(" fileHeader: user header len = " + userHeaderLength + -// ", padding = " + userHeaderLengthPadding); + // System.out.println(" fileHeader: user header len = " + userHeaderLength + + // ", padding = " + userHeaderLengthPadding); -//System.out.println(" fileHeader: total len = " + totalLength); + // System.out.println(" fileHeader: total len = " + totalLength); - userRegister = buffer.getLong(REGISTER1_OFFSET + offset); - trailerPosition = buffer.getLong(TRAILER_POSITION_OFFSET + offset); - userIntFirst = buffer.getInt (INT1_OFFSET + offset); - userIntSecond = buffer.getInt (INT2_OFFSET + offset); + userRegister = buffer.getLong(REGISTER1_OFFSET + offset); + trailerPosition = buffer.getLong(TRAILER_POSITION_OFFSET + offset); + userIntFirst = buffer.getInt(INT1_OFFSET + offset); + userIntSecond = buffer.getInt(INT2_OFFSET + offset); } /** @@ -820,7 +976,8 @@ public void readHeader(ByteBuffer buffer, int offset) throws HipoException { * NOT change. * * @param buffer buffer to read from starting at the beginning. - * @throws HipoException if buffer is not in the proper format or earlier than version 6 + * @throws HipoException if buffer is not in the proper format or earlier than + * version 6 */ public void readHeader(ByteBuffer buffer) throws HipoException { readHeader(buffer, 0); @@ -828,34 +985,35 @@ public void readHeader(ByteBuffer buffer) throws HipoException { /** * Returns a string representation of the record. + * * @return a string representation of the record. */ @Override - public String toString(){ + public String toString() { StringBuilder str = new StringBuilder(); - str.append(String.format("%24s : 0x%x, %s file\n","ID",fileId, - (fileId == EVIO_FILE_UNIQUE_WORD ? "Evio" : "Hipo"))); - str.append(String.format("%24s : %d\n","version",headerVersion)); - str.append(String.format("%24s : %d bytes, words, padding\n","file #",fileNumber)); - str.append(String.format("%24s : %8d / %8d / %8d\n","user header length", - userHeaderLength, userHeaderLengthWords, userHeaderLengthPadding)); - str.append(String.format("%24s : %d\n","header length",headerLength)); - str.append(String.format("%24s : 0x%X\n","magic word",headerMagicWord)); + str.append(String.format("%24s : 0x%x, %s file\n", "ID", fileId, + (fileId == EVIO_FILE_UNIQUE_WORD ? "Evio" : "Hipo"))); + str.append(String.format("%24s : %d\n", "version", headerVersion)); + str.append(String.format("%24s : %d bytes, words, padding\n", "file #", fileNumber)); + str.append(String.format("%24s : %8d / %8d / %8d\n", "user header length", + userHeaderLength, userHeaderLengthWords, userHeaderLengthPadding)); + str.append(String.format("%24s : %d\n", "header length", headerLength)); + str.append(String.format("%24s : 0x%X\n", "magic word", headerMagicWord)); Integer bitInfo = getBitInfoWord(); - str.append(String.format("%24s : %s\n","bit info bits",Integer.toBinaryString(bitInfo))); - str.append(String.format("%24s : 0x%x\n","bit info word",bitInfo)); - str.append(String.format("%24s : %b\n","has dictionary",FileHeader.hasDictionary(bitInfo))); - str.append(String.format("%24s : %b\n","has first event",FileHeader.hasFirstEvent(bitInfo))); - str.append(String.format("%24s : %b\n","has trailer w/ index",FileHeader.hasTrailerWithIndex(bitInfo))); - str.append(String.format("%24s : %d\n","record entries",entries)); + str.append(String.format("%24s : %s\n", "bit info bits", Integer.toBinaryString(bitInfo))); + str.append(String.format("%24s : 0x%x\n", "bit info word", bitInfo)); + str.append(String.format("%24s : %b\n", "has dictionary", FileHeader.hasDictionary(bitInfo))); + str.append(String.format("%24s : %b\n", "has first event", FileHeader.hasFirstEvent(bitInfo))); + str.append(String.format("%24s : %b\n", "has trailer w/ index", FileHeader.hasTrailerWithIndex(bitInfo))); + str.append(String.format("%24s : %d\n", "record entries", entries)); - str.append(String.format("%24s : %d\n"," index length", indexLength)); + str.append(String.format("%24s : %d\n", " index length", indexLength)); - str.append(String.format("%24s : %d\n","trailer position", trailerPosition)); - str.append(String.format("%24s : 0x%X\n","user register", userRegister)); - str.append(String.format("%24s : 0x%X\n","user int #1", userIntFirst)); - str.append(String.format("%24s : 0x%X\n","user int #2", userIntSecond)); + str.append(String.format("%24s : %d\n", "trailer position", trailerPosition)); + str.append(String.format("%24s : 0x%X\n", "user register", userRegister)); + str.append(String.format("%24s : 0x%X\n", "user int #1", userIntFirst)); + str.append(String.format("%24s : 0x%X\n", "user int #2", userIntSecond)); return str.toString(); } @@ -863,39 +1021,45 @@ public String toString(){ /** * Take a string and add padding characters to its left side. * For pretty printing purposes. - * @param original string to modify - * @param pad character to pad with - * @param upTo length to make original string plus padding + * + * @param original string to modify + * @param pad character to pad with + * @param upTo length to make original string plus padding * @return pretty string. */ private static String padLeft(String original, String pad, int upTo) { int npadding = upTo - original.length(); StringBuilder str = new StringBuilder(); - if(npadding>0) for(int i = 0;i < npadding; i++) str.append(pad); + if (npadding > 0) + for (int i = 0; i < npadding; i++) + str.append(pad); str.append(original); return str.toString(); } /** * Print out each word of the given buffer as binary, hex, and decimal. + * * @param buffer buffer to print out. */ public static void byteBufferBinaryString(ByteBuffer buffer) { - int nwords = buffer.array().length/4; - for(int i = 0; i < nwords; i++){ - int value = buffer.getInt(i*4); + int nwords = buffer.array().length / 4; + for (int i = 0; i < nwords; i++) { + int value = buffer.getInt(i * 4); System.out.println(String.format(" word %4d : %36s 0x%8s : %18d", i, - padLeft(Integer.toBinaryString(value),"0",32), - padLeft(Integer.toHexString(value),"0",8), - value)); + padLeft(Integer.toBinaryString(value), "0", 32), + padLeft(Integer.toHexString(value), "0", 8), + value)); } } /** - * Run this class as an executable which tests the writing and reading of a record. + * Run this class as an executable which tests the writing and reading of a + * record. + * * @param args args */ - public static void main(String[] args){ + public static void main(String[] args) { FileHeader header = new FileHeader(); header.setUserHeaderLength(459); @@ -908,21 +1072,20 @@ public static void main(String[] args){ header.setHeaderLength(14); System.out.println(header); - byte[] array = new byte[14*4]; + byte[] array = new byte[14 * 4]; ByteBuffer buffer = ByteBuffer.wrap(array); buffer.order(ByteOrder.LITTLE_ENDIAN); try { header.writeHeader(buffer); - } - catch (HipoException e) {/* never happen */} + } catch (HipoException e) { + /* never happen */} FileHeader header2 = new FileHeader(); try { header2.readHeader(buffer); System.out.println(header2); - } - catch (HipoException e) { + } catch (HipoException e) { e.printStackTrace(); } } diff --git a/java/org/jlab/coda/hipo/HeaderType.java b/src/main/java/org/jlab/coda/hipo/HeaderType.java similarity index 100% rename from java/org/jlab/coda/hipo/HeaderType.java rename to src/main/java/org/jlab/coda/hipo/HeaderType.java diff --git a/java/org/jlab/coda/hipo/HipoException.java b/src/main/java/org/jlab/coda/hipo/HipoException.java similarity index 100% rename from java/org/jlab/coda/hipo/HipoException.java rename to src/main/java/org/jlab/coda/hipo/HipoException.java diff --git a/java/org/jlab/coda/hipo/Reader.java b/src/main/java/org/jlab/coda/hipo/Reader.java similarity index 99% rename from java/org/jlab/coda/hipo/Reader.java rename to src/main/java/org/jlab/coda/hipo/Reader.java index 1d85f753b..a3e9a3a47 100644 --- a/java/org/jlab/coda/hipo/Reader.java +++ b/src/main/java/org/jlab/coda/hipo/Reader.java @@ -1817,9 +1817,9 @@ public void scanFile(boolean force) throws IOException, HipoException { // Index in trailer gets first priority. // Index in file header gets next priority. boolean fileHasIndex = fileHeader.hasTrailerWithIndex() || (fileHeader.hasIndex()); -//System.out.println(" file has index = " + fileHasIndex -// + ", has trailer w/ index = " + fileHeader.hasTrailerWithIndex() -// + ", file hdr has index = " + fileHeader.hasIndex()); + //System.out.println(" file has index = " + fileHasIndex + // + ", has trailer w/ index = " + fileHeader.hasTrailerWithIndex() + // + ", file hdr has index = " + fileHeader.hasIndex()); // If there is no index, scan file if (!fileHasIndex) { diff --git a/java/org/jlab/coda/hipo/RecordHeader.java b/src/main/java/org/jlab/coda/hipo/RecordHeader.java similarity index 100% rename from java/org/jlab/coda/hipo/RecordHeader.java rename to src/main/java/org/jlab/coda/hipo/RecordHeader.java diff --git a/java/org/jlab/coda/hipo/RecordInputStream.java b/src/main/java/org/jlab/coda/hipo/RecordInputStream.java similarity index 100% rename from java/org/jlab/coda/hipo/RecordInputStream.java rename to src/main/java/org/jlab/coda/hipo/RecordInputStream.java diff --git a/java/org/jlab/coda/hipo/RecordOutputStream.java b/src/main/java/org/jlab/coda/hipo/RecordOutputStream.java similarity index 100% rename from java/org/jlab/coda/hipo/RecordOutputStream.java rename to src/main/java/org/jlab/coda/hipo/RecordOutputStream.java diff --git a/java/org/jlab/coda/hipo/RecordRingItem.java b/src/main/java/org/jlab/coda/hipo/RecordRingItem.java similarity index 100% rename from java/org/jlab/coda/hipo/RecordRingItem.java rename to src/main/java/org/jlab/coda/hipo/RecordRingItem.java diff --git a/java/org/jlab/coda/hipo/RecordSupply.java b/src/main/java/org/jlab/coda/hipo/RecordSupply.java similarity index 100% rename from java/org/jlab/coda/hipo/RecordSupply.java rename to src/main/java/org/jlab/coda/hipo/RecordSupply.java diff --git a/java/org/jlab/coda/hipo/Writer.java b/src/main/java/org/jlab/coda/hipo/Writer.java similarity index 100% rename from java/org/jlab/coda/hipo/Writer.java rename to src/main/java/org/jlab/coda/hipo/Writer.java diff --git a/java/org/jlab/coda/hipo/WriterMT.java b/src/main/java/org/jlab/coda/hipo/WriterMT.java similarity index 100% rename from java/org/jlab/coda/hipo/WriterMT.java rename to src/main/java/org/jlab/coda/hipo/WriterMT.java diff --git a/java/org/jlab/coda/jevio/BankHeader.java b/src/main/java/org/jlab/coda/jevio/BankHeader.java similarity index 100% rename from java/org/jlab/coda/jevio/BankHeader.java rename to src/main/java/org/jlab/coda/jevio/BankHeader.java diff --git a/java/org/jlab/coda/jevio/BaseStructure.java b/src/main/java/org/jlab/coda/jevio/BaseStructure.java similarity index 100% rename from java/org/jlab/coda/jevio/BaseStructure.java rename to src/main/java/org/jlab/coda/jevio/BaseStructure.java diff --git a/java/org/jlab/coda/jevio/BaseStructureHeader.java b/src/main/java/org/jlab/coda/jevio/BaseStructureHeader.java similarity index 100% rename from java/org/jlab/coda/jevio/BaseStructureHeader.java rename to src/main/java/org/jlab/coda/jevio/BaseStructureHeader.java diff --git a/java/org/jlab/coda/jevio/BlockHeaderV2.java b/src/main/java/org/jlab/coda/jevio/BlockHeaderV2.java similarity index 100% rename from java/org/jlab/coda/jevio/BlockHeaderV2.java rename to src/main/java/org/jlab/coda/jevio/BlockHeaderV2.java diff --git a/java/org/jlab/coda/jevio/BlockHeaderV4.java b/src/main/java/org/jlab/coda/jevio/BlockHeaderV4.java similarity index 100% rename from java/org/jlab/coda/jevio/BlockHeaderV4.java rename to src/main/java/org/jlab/coda/jevio/BlockHeaderV4.java diff --git a/java/org/jlab/coda/jevio/BlockNode.java b/src/main/java/org/jlab/coda/jevio/BlockNode.java similarity index 100% rename from java/org/jlab/coda/jevio/BlockNode.java rename to src/main/java/org/jlab/coda/jevio/BlockNode.java diff --git a/java/org/jlab/coda/jevio/ByteDataTransformer.java b/src/main/java/org/jlab/coda/jevio/ByteDataTransformer.java similarity index 100% rename from java/org/jlab/coda/jevio/ByteDataTransformer.java rename to src/main/java/org/jlab/coda/jevio/ByteDataTransformer.java diff --git a/java/org/jlab/coda/jevio/CompactEventBuilder.java b/src/main/java/org/jlab/coda/jevio/CompactEventBuilder.java similarity index 100% rename from java/org/jlab/coda/jevio/CompactEventBuilder.java rename to src/main/java/org/jlab/coda/jevio/CompactEventBuilder.java diff --git a/java/org/jlab/coda/jevio/CompositeData.java b/src/main/java/org/jlab/coda/jevio/CompositeData.java similarity index 100% rename from java/org/jlab/coda/jevio/CompositeData.java rename to src/main/java/org/jlab/coda/jevio/CompositeData.java diff --git a/java/org/jlab/coda/jevio/DataType.java b/src/main/java/org/jlab/coda/jevio/DataType.java similarity index 100% rename from java/org/jlab/coda/jevio/DataType.java rename to src/main/java/org/jlab/coda/jevio/DataType.java diff --git a/java/org/jlab/coda/jevio/Environment.java b/src/main/java/org/jlab/coda/jevio/Environment.java similarity index 100% rename from java/org/jlab/coda/jevio/Environment.java rename to src/main/java/org/jlab/coda/jevio/Environment.java diff --git a/java/org/jlab/coda/jevio/EventBuilder.java b/src/main/java/org/jlab/coda/jevio/EventBuilder.java similarity index 100% rename from java/org/jlab/coda/jevio/EventBuilder.java rename to src/main/java/org/jlab/coda/jevio/EventBuilder.java diff --git a/java/org/jlab/coda/jevio/EventParser.java b/src/main/java/org/jlab/coda/jevio/EventParser.java similarity index 100% rename from java/org/jlab/coda/jevio/EventParser.java rename to src/main/java/org/jlab/coda/jevio/EventParser.java diff --git a/java/org/jlab/coda/jevio/EventWriter.java b/src/main/java/org/jlab/coda/jevio/EventWriter.java similarity index 100% rename from java/org/jlab/coda/jevio/EventWriter.java rename to src/main/java/org/jlab/coda/jevio/EventWriter.java diff --git a/java/org/jlab/coda/jevio/EventWriterUnsync.java b/src/main/java/org/jlab/coda/jevio/EventWriterUnsync.java similarity index 100% rename from java/org/jlab/coda/jevio/EventWriterUnsync.java rename to src/main/java/org/jlab/coda/jevio/EventWriterUnsync.java diff --git a/java/org/jlab/coda/jevio/EventWriterUnsyncV4.java b/src/main/java/org/jlab/coda/jevio/EventWriterUnsyncV4.java similarity index 100% rename from java/org/jlab/coda/jevio/EventWriterUnsyncV4.java rename to src/main/java/org/jlab/coda/jevio/EventWriterUnsyncV4.java diff --git a/java/org/jlab/coda/jevio/EventWriterV4.java b/src/main/java/org/jlab/coda/jevio/EventWriterV4.java similarity index 100% rename from java/org/jlab/coda/jevio/EventWriterV4.java rename to src/main/java/org/jlab/coda/jevio/EventWriterV4.java diff --git a/java/org/jlab/coda/jevio/EvioBank.java b/src/main/java/org/jlab/coda/jevio/EvioBank.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioBank.java rename to src/main/java/org/jlab/coda/jevio/EvioBank.java diff --git a/java/org/jlab/coda/jevio/EvioByteArrayOutputStream.java b/src/main/java/org/jlab/coda/jevio/EvioByteArrayOutputStream.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioByteArrayOutputStream.java rename to src/main/java/org/jlab/coda/jevio/EvioByteArrayOutputStream.java diff --git a/java/org/jlab/coda/jevio/EvioCompactReader.java b/src/main/java/org/jlab/coda/jevio/EvioCompactReader.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioCompactReader.java rename to src/main/java/org/jlab/coda/jevio/EvioCompactReader.java diff --git a/java/org/jlab/coda/jevio/EvioCompactReaderUnsyncV4.java b/src/main/java/org/jlab/coda/jevio/EvioCompactReaderUnsyncV4.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioCompactReaderUnsyncV4.java rename to src/main/java/org/jlab/coda/jevio/EvioCompactReaderUnsyncV4.java diff --git a/java/org/jlab/coda/jevio/EvioCompactReaderUnsyncV6.java b/src/main/java/org/jlab/coda/jevio/EvioCompactReaderUnsyncV6.java similarity index 95% rename from java/org/jlab/coda/jevio/EvioCompactReaderUnsyncV6.java rename to src/main/java/org/jlab/coda/jevio/EvioCompactReaderUnsyncV6.java index 4ad548dcf..d38d481d1 100644 --- a/java/org/jlab/coda/jevio/EvioCompactReaderUnsyncV6.java +++ b/src/main/java/org/jlab/coda/jevio/EvioCompactReaderUnsyncV6.java @@ -10,6 +10,7 @@ package org.jlab.coda.jevio; +import org.jlab.coda.hipo.HeaderType; import org.jlab.coda.hipo.HipoException; import org.jlab.coda.hipo.Reader; @@ -85,6 +86,10 @@ public EvioCompactReaderUnsyncV6(ByteBuffer byteBuffer, EvioNodeSource pool) if (!reader.isEvioFormat()) { throw new EvioException("buffer not in evio format"); } + // if (reader.getFileHeader().getHeaderType() == HeaderType.HIPO_FILE) { + // throw new EvioException("file not in evio format"); + // } + } catch (HipoException e) { throw new EvioException(e); diff --git a/java/org/jlab/coda/jevio/EvioCompactReaderV4.java b/src/main/java/org/jlab/coda/jevio/EvioCompactReaderV4.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioCompactReaderV4.java rename to src/main/java/org/jlab/coda/jevio/EvioCompactReaderV4.java diff --git a/java/org/jlab/coda/jevio/EvioCompactReaderV6.java b/src/main/java/org/jlab/coda/jevio/EvioCompactReaderV6.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioCompactReaderV6.java rename to src/main/java/org/jlab/coda/jevio/EvioCompactReaderV6.java diff --git a/java/org/jlab/coda/jevio/EvioCompactStructureHandler.java b/src/main/java/org/jlab/coda/jevio/EvioCompactStructureHandler.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioCompactStructureHandler.java rename to src/main/java/org/jlab/coda/jevio/EvioCompactStructureHandler.java diff --git a/java/org/jlab/coda/jevio/EvioDictionaryEntry.java b/src/main/java/org/jlab/coda/jevio/EvioDictionaryEntry.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioDictionaryEntry.java rename to src/main/java/org/jlab/coda/jevio/EvioDictionaryEntry.java diff --git a/java/org/jlab/coda/jevio/EvioDictionaryEntryType.java b/src/main/java/org/jlab/coda/jevio/EvioDictionaryEntryType.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioDictionaryEntryType.java rename to src/main/java/org/jlab/coda/jevio/EvioDictionaryEntryType.java diff --git a/java/org/jlab/coda/jevio/EvioEvent.java b/src/main/java/org/jlab/coda/jevio/EvioEvent.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioEvent.java rename to src/main/java/org/jlab/coda/jevio/EvioEvent.java diff --git a/java/org/jlab/coda/jevio/EvioException.java b/src/main/java/org/jlab/coda/jevio/EvioException.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioException.java rename to src/main/java/org/jlab/coda/jevio/EvioException.java diff --git a/java/org/jlab/coda/jevio/EvioNode.java b/src/main/java/org/jlab/coda/jevio/EvioNode.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioNode.java rename to src/main/java/org/jlab/coda/jevio/EvioNode.java diff --git a/java/org/jlab/coda/jevio/EvioNodePool.java b/src/main/java/org/jlab/coda/jevio/EvioNodePool.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioNodePool.java rename to src/main/java/org/jlab/coda/jevio/EvioNodePool.java diff --git a/java/org/jlab/coda/jevio/EvioNodeSource.java b/src/main/java/org/jlab/coda/jevio/EvioNodeSource.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioNodeSource.java rename to src/main/java/org/jlab/coda/jevio/EvioNodeSource.java diff --git a/java/org/jlab/coda/jevio/EvioReader.java b/src/main/java/org/jlab/coda/jevio/EvioReader.java similarity index 96% rename from java/org/jlab/coda/jevio/EvioReader.java rename to src/main/java/org/jlab/coda/jevio/EvioReader.java index 83b58b266..4bffadde7 100644 --- a/java/org/jlab/coda/jevio/EvioReader.java +++ b/src/main/java/org/jlab/coda/jevio/EvioReader.java @@ -53,6 +53,12 @@ public class EvioReader implements IEvioReader { */ private ByteOrder byteOrder; + /** If enabled by setting > 0, then scan +/- this many words around expected + * magic number when reading record header + */ + private int magicNumRecoverWin = -1; + + /** The buffer being read. */ private final ByteBuffer byteBuffer; @@ -327,6 +333,9 @@ public void setBuffer(ByteBuffer buf) throws EvioException, IOException { /** {@inheritDoc} */ public boolean checkBlockNumberSequence() {return reader.checkBlockNumberSequence();} + /** {@inheritDoc} */ + public void addHeaderRecoveryCheck() {reader.addHeaderRecoveryCheck();} + /** {@inheritDoc} */ public ByteOrder getByteOrder() {return reader.getByteOrder();} diff --git a/java/org/jlab/coda/jevio/EvioReaderUnsyncV4.java b/src/main/java/org/jlab/coda/jevio/EvioReaderUnsyncV4.java similarity index 91% rename from java/org/jlab/coda/jevio/EvioReaderUnsyncV4.java rename to src/main/java/org/jlab/coda/jevio/EvioReaderUnsyncV4.java index 0fc5d594a..580f9c920 100644 --- a/java/org/jlab/coda/jevio/EvioReaderUnsyncV4.java +++ b/src/main/java/org/jlab/coda/jevio/EvioReaderUnsyncV4.java @@ -125,6 +125,13 @@ public class EvioReaderUnsyncV4 implements IEvioReader { /** Block number expected when reading. Used to check sequence of blocks. */ protected int blockNumberExpected = 1; + /** If true, attempt header recovery when block headers aren’t where expected + * currently, having extraneous words is supported + * it doesn't handle missing header words yet (more difficult) + * also ignores lastBlock flag + */ + protected boolean doHeaderRecoveryCheck = false; + /** If true, throw an exception if block numbers are out of sequence. */ protected boolean checkBlockNumberSequence; @@ -612,6 +619,13 @@ public void setBuffer(ByteBuffer buf) throws EvioException, IOException { @Override public boolean checkBlockNumberSequence() { return checkBlockNumberSequence; } + /** Invoke to try header recovery when block headers aren’t where expected + * currently, having extraneous words is supported + * it doesn't handle missing header words yet (more difficult) + */ + @Override + public void addHeaderRecoveryCheck() {this.doHeaderRecoveryCheck=true;} + /** * Get the byte order of the file/buffer being read. * @return byte order of the file/buffer being read. @@ -971,7 +985,7 @@ protected void prepareForBufferRead(ByteBuffer buffer) { protected ReadStatus processNextBlock() throws IOException { // We already read the last block header - if (lastBlock) { + if (lastBlock && !doHeaderRecoveryCheck) { return ReadStatus.END_OF_FILE; } @@ -1023,12 +1037,65 @@ else if (bytesInBuf % 32768 == 0) { return ReadStatus.END_OF_FILE; } + long initPos = fileChannel.position(); + // Read len of block in 32 bit words int blkSize = dataStream.readInt(); if (swap) blkSize = Integer.reverseBytes(blkSize); - // Change to bytes - int blkBytes = 4 * blkSize; + // Check block size, attempt to recover if flag set + // (otherwise return exception with a hint to set flag) + // System.out.println("blkSize BEFORE = " + blkSize); + if(doHeaderRecoveryCheck && fileSize - fileChannel.position() >= 10*4) { + + int expectedMagicPos = 27; // in words + int words_to_skip = 0; // words_to_skip = foundMagicPos - expectedMagicPos + + // peek at 40 words around current position (32-bit ints, 4 bytes) + // WITHOUT changing current position + ByteBuffer peek = ByteBuffer.allocate(40*4).order(byteOrder); + // Perform positional read (does NOT move file position) + int bytesRead = fileChannel.read(peek, initPos - 20 * 4); + peek.flip(); // prepare for reading + if (bytesRead != 40 * 4) { + throw new IOException("Failed to read sufficient data for header recovery."); + } + + int foundMagicPos = -1000; // dummy val + int[] words = new int[40]; + for (int i=0; i < 40; i++) { + words[i] = peek.getInt(); + // System.out.println(" peeked word " + i + " = 0x" + Integer.toHexString(words[i])); + if(words[i] == IBlockHeader.MAGIC_NUMBER) { + foundMagicPos = i; + // System.out.println("Found magic # at pos " + foundMagicPos); + } + } + words_to_skip = foundMagicPos-expectedMagicPos; + + // It's a little more challenging to look backwards than forwards + // (since e rely on a relative read just below) + // Not sure why, but trying to reset position doesn't work + if (words_to_skip <= -1) { + System.out.println("Error: According to magic " + + "word, block header began earlier than the current file position"); + // until implemented, return exception + return ReadStatus.EVIO_EXCEPTION; + } + if (words_to_skip > 0) { + // Skip over some this many words + for (int i=0; i < (words_to_skip); i++) { + blkSize = dataStream.readInt(); + if (swap) blkSize = Integer.reverseBytes(blkSize); + } + } + blkSize+=words_to_skip; // Previous block pointed us to wrong place, + // but this will fix for future blocks + } + + // Change to bytes + int blkBytes = 4 * (blkSize); + // Enough data left to read rest of block? if (fileSize - fileChannel.position() < blkBytes-4) { return ReadStatus.END_OF_FILE; @@ -1071,6 +1138,7 @@ else if (bytesInBuf % 32768 == 0) { } if (evioVersion >= 4) { + // Read the header data. blockHeader4.setSize(byteBuffer.getInt()); blockHeader4.setNumber(byteBuffer.getInt()); @@ -1085,6 +1153,15 @@ else if (bytesInBuf % 32768 == 0) { blockHeader4.setMagicNumber(byteBuffer.getInt()); blockHeader = blockHeader4; + // System.out.println("BlockHeader v4:"); + // System.out.println(" block length = " + blockHeader4.getSize()); + // System.out.println(" block number = " + blockHeader4.getNumber()); + // System.out.println(" header length = " + blockHeader4.getHeaderLength()); + // System.out.println(" event count = " + blockHeader4.getEventCount()); + // System.out.println(" version = " + blockHeader4.getVersion()); + // System.out.println(" has Dict = " + blockHeader4.getBitInfo(0)); + // System.out.println(" is End = " + lastBlock); + // Deal with non-standard header lengths here int headerLenDiff = blockHeader4.getHeaderLength() - BlockHeaderV4.HEADER_SIZE; // If too small quit with error since headers have a minimum size @@ -1097,6 +1174,7 @@ else if (headerLenDiff > 0) { byteBuffer.getInt(); } } + } else if (evioVersion < 4) { // read the header data @@ -1121,9 +1199,8 @@ else if (evioVersion < 4) { if (checkBlockNumberSequence) { if (blockHeader.getNumber() != blockNumberExpected) { -System.out.println("block # out of sequence, got " + blockHeader.getNumber() + - " expecting " + blockNumberExpected); - + System.out.println("block # out of sequence, got " + blockHeader.getNumber() + + " expecting " + blockNumberExpected); return ReadStatus.EVIO_EXCEPTION; } blockNumberExpected++; @@ -1134,7 +1211,7 @@ else if (evioVersion < 4) { return ReadStatus.EVIO_EXCEPTION; } catch (BufferUnderflowException a) { -System.err.println("ERROR endOfBuffer " + a); + System.err.println("ERROR endOfBuffer " + a); byteBuffer.clear(); return ReadStatus.UNKNOWN_ERROR; } diff --git a/java/org/jlab/coda/jevio/EvioReaderUnsyncV6.java b/src/main/java/org/jlab/coda/jevio/EvioReaderUnsyncV6.java similarity index 91% rename from java/org/jlab/coda/jevio/EvioReaderUnsyncV6.java rename to src/main/java/org/jlab/coda/jevio/EvioReaderUnsyncV6.java index 3b20f2e20..c8780e9bf 100644 --- a/java/org/jlab/coda/jevio/EvioReaderUnsyncV6.java +++ b/src/main/java/org/jlab/coda/jevio/EvioReaderUnsyncV6.java @@ -10,6 +10,7 @@ package org.jlab.coda.jevio; +import org.jlab.coda.hipo.HeaderType; import org.jlab.coda.hipo.HipoException; import org.jlab.coda.hipo.Reader; @@ -60,6 +61,12 @@ public class EvioReaderUnsyncV6 implements IEvioReader { /** Parser object for file/buffer. */ protected EventParser parser; + /** If true, attempt header recovery when block headers aren’t where expected + * currently, having extraneous words is supported + * it doesn't handle missing header words yet (more difficult) + */ + protected boolean doHeaderRecoveryCheck = false; + //------------------------ @@ -127,6 +134,10 @@ public EvioReaderUnsyncV6(File file) throws EvioException, IOException { try { reader = new Reader(file.getPath(), checkRecNumSeq, true); parser = new EventParser(); + + // if (reader.getFileHeader().getHeaderType() == HeaderType.HIPO_FILE) { + // throw new EvioException("file not in evio format"); + // } } catch (HipoException e) { throw new EvioException(e); @@ -164,6 +175,9 @@ public EvioReaderUnsyncV6(ByteBuffer byteBuffer, boolean checkRecNumSeq) throws if (!reader.isEvioFormat()) { throw new EvioException("buffer not in evio format"); } + // if (reader.getFileHeader().getHeaderType() == HeaderType.HIPO_FILE) { + // throw new EvioException("file not in evio format"); + // } parser = new EventParser(); } catch (HipoException e) { @@ -201,6 +215,13 @@ public void setBuffer(ByteBuffer buf) throws EvioException, IOException { @Override public boolean checkBlockNumberSequence() {return reader.getCheckRecordNumberSequence();} + /** Invoke to try header recovery when block headers aren’t where expected + * currently, having extraneous words is supported + * it doesn't handle missing header words yet (more difficult) + */ + @Override + public void addHeaderRecoveryCheck() {this.doHeaderRecoveryCheck=true;} + /** {@inheritDoc} */ @Override public ByteOrder getByteOrder() {return reader.getByteOrder();} diff --git a/java/org/jlab/coda/jevio/EvioReaderV4.java b/src/main/java/org/jlab/coda/jevio/EvioReaderV4.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioReaderV4.java rename to src/main/java/org/jlab/coda/jevio/EvioReaderV4.java diff --git a/java/org/jlab/coda/jevio/EvioReaderV6.java b/src/main/java/org/jlab/coda/jevio/EvioReaderV6.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioReaderV6.java rename to src/main/java/org/jlab/coda/jevio/EvioReaderV6.java diff --git a/java/org/jlab/coda/jevio/EvioSegment.java b/src/main/java/org/jlab/coda/jevio/EvioSegment.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioSegment.java rename to src/main/java/org/jlab/coda/jevio/EvioSegment.java diff --git a/java/org/jlab/coda/jevio/EvioTagSegment.java b/src/main/java/org/jlab/coda/jevio/EvioTagSegment.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioTagSegment.java rename to src/main/java/org/jlab/coda/jevio/EvioTagSegment.java diff --git a/java/org/jlab/coda/jevio/EvioXMLDictionary.java b/src/main/java/org/jlab/coda/jevio/EvioXMLDictionary.java similarity index 100% rename from java/org/jlab/coda/jevio/EvioXMLDictionary.java rename to src/main/java/org/jlab/coda/jevio/EvioXMLDictionary.java diff --git a/java/org/jlab/coda/jevio/IBlockHeader.java b/src/main/java/org/jlab/coda/jevio/IBlockHeader.java similarity index 100% rename from java/org/jlab/coda/jevio/IBlockHeader.java rename to src/main/java/org/jlab/coda/jevio/IBlockHeader.java diff --git a/java/org/jlab/coda/jevio/IEvioCompactReader.java b/src/main/java/org/jlab/coda/jevio/IEvioCompactReader.java similarity index 100% rename from java/org/jlab/coda/jevio/IEvioCompactReader.java rename to src/main/java/org/jlab/coda/jevio/IEvioCompactReader.java diff --git a/java/org/jlab/coda/jevio/IEvioFilter.java b/src/main/java/org/jlab/coda/jevio/IEvioFilter.java similarity index 100% rename from java/org/jlab/coda/jevio/IEvioFilter.java rename to src/main/java/org/jlab/coda/jevio/IEvioFilter.java diff --git a/java/org/jlab/coda/jevio/IEvioListener.java b/src/main/java/org/jlab/coda/jevio/IEvioListener.java similarity index 100% rename from java/org/jlab/coda/jevio/IEvioListener.java rename to src/main/java/org/jlab/coda/jevio/IEvioListener.java diff --git a/java/org/jlab/coda/jevio/IEvioProgressListener.java b/src/main/java/org/jlab/coda/jevio/IEvioProgressListener.java similarity index 100% rename from java/org/jlab/coda/jevio/IEvioProgressListener.java rename to src/main/java/org/jlab/coda/jevio/IEvioProgressListener.java diff --git a/java/org/jlab/coda/jevio/IEvioReader.java b/src/main/java/org/jlab/coda/jevio/IEvioReader.java similarity index 98% rename from java/org/jlab/coda/jevio/IEvioReader.java rename to src/main/java/org/jlab/coda/jevio/IEvioReader.java index 93d2b2bfc..e22c0c129 100644 --- a/java/org/jlab/coda/jevio/IEvioReader.java +++ b/src/main/java/org/jlab/coda/jevio/IEvioReader.java @@ -39,6 +39,12 @@ public interface IEvioReader { */ boolean checkBlockNumberSequence(); + /** Invoke to try header recovery when block headers aren’t where expected + * currently, having extraneous words is supported + * it doesn't handle missing header words yet (more difficult) + */ + void addHeaderRecoveryCheck(); + /** * Get the byte order of the file/buffer being read. * @return byte order of the file/buffer being read. diff --git a/java/org/jlab/coda/jevio/IEvioStructure.java b/src/main/java/org/jlab/coda/jevio/IEvioStructure.java similarity index 100% rename from java/org/jlab/coda/jevio/IEvioStructure.java rename to src/main/java/org/jlab/coda/jevio/IEvioStructure.java diff --git a/java/org/jlab/coda/jevio/IEvioWriter.java b/src/main/java/org/jlab/coda/jevio/IEvioWriter.java similarity index 100% rename from java/org/jlab/coda/jevio/IEvioWriter.java rename to src/main/java/org/jlab/coda/jevio/IEvioWriter.java diff --git a/java/org/jlab/coda/jevio/INameProvider.java b/src/main/java/org/jlab/coda/jevio/INameProvider.java similarity index 100% rename from java/org/jlab/coda/jevio/INameProvider.java rename to src/main/java/org/jlab/coda/jevio/INameProvider.java diff --git a/java/org/jlab/coda/jevio/MappedMemoryHandler.java b/src/main/java/org/jlab/coda/jevio/MappedMemoryHandler.java similarity index 100% rename from java/org/jlab/coda/jevio/MappedMemoryHandler.java rename to src/main/java/org/jlab/coda/jevio/MappedMemoryHandler.java diff --git a/java/org/jlab/coda/jevio/NameProvider.java b/src/main/java/org/jlab/coda/jevio/NameProvider.java similarity index 100% rename from java/org/jlab/coda/jevio/NameProvider.java rename to src/main/java/org/jlab/coda/jevio/NameProvider.java diff --git a/java/org/jlab/coda/jevio/NameProviderFactory.java b/src/main/java/org/jlab/coda/jevio/NameProviderFactory.java similarity index 100% rename from java/org/jlab/coda/jevio/NameProviderFactory.java rename to src/main/java/org/jlab/coda/jevio/NameProviderFactory.java diff --git a/java/org/jlab/coda/jevio/ReadStatus.java b/src/main/java/org/jlab/coda/jevio/ReadStatus.java similarity index 100% rename from java/org/jlab/coda/jevio/ReadStatus.java rename to src/main/java/org/jlab/coda/jevio/ReadStatus.java diff --git a/java/org/jlab/coda/jevio/SegmentHeader.java b/src/main/java/org/jlab/coda/jevio/SegmentHeader.java similarity index 100% rename from java/org/jlab/coda/jevio/SegmentHeader.java rename to src/main/java/org/jlab/coda/jevio/SegmentHeader.java diff --git a/java/org/jlab/coda/jevio/StructureFinder.java b/src/main/java/org/jlab/coda/jevio/StructureFinder.java similarity index 100% rename from java/org/jlab/coda/jevio/StructureFinder.java rename to src/main/java/org/jlab/coda/jevio/StructureFinder.java diff --git a/java/org/jlab/coda/jevio/StructureTransformer.java b/src/main/java/org/jlab/coda/jevio/StructureTransformer.java similarity index 100% rename from java/org/jlab/coda/jevio/StructureTransformer.java rename to src/main/java/org/jlab/coda/jevio/StructureTransformer.java diff --git a/java/org/jlab/coda/jevio/StructureType.java b/src/main/java/org/jlab/coda/jevio/StructureType.java similarity index 100% rename from java/org/jlab/coda/jevio/StructureType.java rename to src/main/java/org/jlab/coda/jevio/StructureType.java diff --git a/java/org/jlab/coda/jevio/TagSegmentHeader.java b/src/main/java/org/jlab/coda/jevio/TagSegmentHeader.java similarity index 100% rename from java/org/jlab/coda/jevio/TagSegmentHeader.java rename to src/main/java/org/jlab/coda/jevio/TagSegmentHeader.java diff --git a/java/org/jlab/coda/jevio/Utilities.java b/src/main/java/org/jlab/coda/jevio/Utilities.java similarity index 100% rename from java/org/jlab/coda/jevio/Utilities.java rename to src/main/java/org/jlab/coda/jevio/Utilities.java diff --git a/java/org/jlab/coda/jevio/WriteStatus.java b/src/main/java/org/jlab/coda/jevio/WriteStatus.java similarity index 100% rename from java/org/jlab/coda/jevio/WriteStatus.java rename to src/main/java/org/jlab/coda/jevio/WriteStatus.java diff --git a/src/main/java/org/jlab/coda/jevio/dev/EvioReadFileSimple.java b/src/main/java/org/jlab/coda/jevio/dev/EvioReadFileSimple.java new file mode 100644 index 000000000..87ad6b312 --- /dev/null +++ b/src/main/java/org/jlab/coda/jevio/dev/EvioReadFileSimple.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2014, Jefferson Science Associates + * + * Thomas Jefferson National Accelerator Facility + * Data Acquisition Group + * + * 12000, Jefferson Ave, Newport News, VA 23606 + * Phone : (757)-269-7100 + * + */ + +package org.jlab.coda.jevio.dev; + +import java.io.IOException; + +import org.jlab.coda.jevio.EvioException; +import org.jlab.coda.jevio.EvioReader; + +public class EvioReadFileSimple { + + public static void main(String[] args) { + // This is a simple example of how to read an Evio file. + // It assumes that the Evio file is already created and contains events. + + if (args.length != 1) { + System.out.println("Usage: java EvioReadFileSimple "); + return; + } + + String filename = args[0]; // Replace with your Evio file name + + try { + EvioReader reader = new EvioReader(filename); + reader.addHeaderRecoveryCheck(); + + int evCount = reader.getEventCount(); + System.out.println("Number of events in file: " + evCount); + + reader.close(); + } catch (EvioException | IOException e) { + e.printStackTrace(); + } + } + +} diff --git a/src/test/ReadWriteV4Test.cpp b/src/test/ReadWriteV4Test.cpp deleted file mode 100644 index ce1a5a83e..000000000 --- a/src/test/ReadWriteV4Test.cpp +++ /dev/null @@ -1,381 +0,0 @@ -/** - * Copyright (c) 2024, Jefferson Science Associates - * - * Thomas Jefferson National Accelerator Facility - * Data Acquisition Group - * - * 12000, Jefferson Ave, Newport News, VA 23606 - * Phone : (757)-269-7100 - * - * @date 07/25/2024 - * @author timmer - */ - - - -#include -#include -#include -#include - - -#include "TestBase.h" - - -using namespace std; - - -namespace evio { - - - class ReadWriteTest : public TestBase { - - - public: - - - - std::shared_ptr generateEvioBank(ByteOrder & order, uint16_t tag, uint8_t num) { - // Event, traditional bank of banks - EventBuilder builder(tag, DataType::BANK, num); - auto ev = builder.getEvent(); - - // add a bank of doubles - auto bank1 = EvioBank::getInstance(22, DataType::DOUBLE64, 0); - double dData[1000]; - for (int i = 0; i < 1000; i++) { - dData[i] = i + 1.; - } - builder.appendDoubleData(bank1, dData, 1000); - cout << " generate Evio Bank, bank1 len = " << bank1->getTotalBytes() << endl; - builder.addChild(ev, bank1); - cout << " generate Evio Bank, ev len = " << ev->getTotalBytes() << endl; - - return static_cast>(ev); - } - - - - void writeFile(string finalFilename, uint16_t tag, uint8_t num) { - - std::cout << std::endl; - std::cout << "--------------------------------------------" << std::endl; - std::cout << "---------- Write to file --------------" << std::endl; - std::cout << "--------------------------------------------" << std::endl; - - ByteOrder outputOrder = ByteOrder::ENDIAN_LITTLE; - - // Create a "first event" - uint32_t firstEventData[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - uint32_t firstEventDataLen = 10; - EventBuilder builder(1, DataType::UINT32, 2); - std::shared_ptr firstEvent = builder.getEvent(); - builder.appendUIntData(firstEvent, firstEventData, firstEventDataLen); - //auto firstBank = static_cast>(firstEvent); - - std::string directory; - std::string runType; - - // Create files - EventWriterV4 writer(finalFilename, directory, runType, 1, 0, - EventWriterV4::DEFAULT_BLOCK_SIZE, - EventWriterV4::DEFAULT_BLOCK_COUNT, - outputOrder, dictionary, - true, false, - firstEvent); - - -// EventWriterV4(std::string & baseName, -// const std::string & directory, const std::string & runType, -// uint32_t runNumber = 1, uint64_t split = 0, -// uint32_t maxBlockSize = DEFAULT_BLOCK_SIZE, -// uint32_t maxEventCount = DEFAULT_BLOCK_COUNT, -// const ByteOrder & byteOrder = ByteOrder::nativeOrder(), -// const std::string & xmlDictionary = "", -// bool overWriteOK = true, bool append = false, -// std::shared_ptr firstEvent = nullptr, -// uint32_t streamId = 0, uint32_t splitNumber = 0, -// uint32_t splitIncrement = 1, uint32_t streamCount = 1, -// uint32_t bufferSize = DEFAULT_BUFFER_SIZE, -// std::bitset<24> *bitInfo = nullptr); - - - // Create an event with lots of stuff in it - auto evioDataBuf = createCompactEventBuffer(tag, num, outputOrder); - - // Create node from this buffer - std::shared_ptr node = EvioNode::extractEventNode(evioDataBuf,0,0,0); - - // Create EvioBank - std::shared_ptr bank = generateEvioBank(outputOrder, tag, num); - - // write as buffer - writer.writeEvent(evioDataBuf, false); - cout << " Wrote evio buffer, len = " << evioDataBuf->limit() << endl; - - // write as node - writer.writeEvent(node, false); - cout << " Wrote evio node, total bytes = " << node->getTotalBytes() << endl; - - // write as EvioBank - writer.writeEvent(bank); - cout << " Wrote evio bank, total bytes = " << bank->getTotalBytes() << endl; - cout << " Wrote evio bank, header len in bytes = " << 4*(bank->getHeader()->getLength() + 1) << endl; - - writer.close(); - cout << "Finished writing file " << finalFilename << " now read it" << endl; - - //Util::printBytes(finalFilename, 0, 2000, "File bytes"); - } - - - void readFile(string finalFilename) { - - std::cout << std::endl; - std::cout << "--------------------------------------------" << std::endl; - std::cout << "---------- Read from file --------------" << std::endl; - std::cout << "--------------------------------------------" << std::endl; - - EvioReader reader(finalFilename); - ByteOrder & order = reader.getByteOrder(); - - cout << "Read in file " << finalFilename << " of byte order " << order.getName() << endl; - int32_t evCount = reader.getEventCount(); - cout << "Got " << evCount << " events" << endl; - - string dict = reader.getDictionaryXML(); - if (dict.empty()) { - cout << "\nNo dictionary" << endl; - } - else { - cout << "\nGot dictionary:\n" << dict << endl; - } - - auto pFE = reader.getFirstEvent(); - if (pFE != nullptr) { - cout << "\nGot first Event:\n" << pFE->toString() << endl << endl; - } - else { - cout << "\nNo first event" << endl; - } - - cout << "Print out regular events:" << endl; - - for (int i=0; i < evCount; i++) { - auto ev = reader.getEvent(i+1); - cout << "\nEvent" << (i+1) << ":\n" << ev->toString() << endl; - } - } - - - - void writeAndReadBuffer(uint16_t tag, uint8_t num) { - - std::cout << std::endl; - std::cout << "--------------------------------------------" << std::endl; - std::cout << "---------- Write to buf --------------" << std::endl; - std::cout << "--------------------------------------------" << std::endl; - - ByteOrder order = ByteOrder::ENDIAN_LITTLE; - std::shared_ptr buffer = std::make_shared(200000); - buffer->order(order); - - // Create a "first event" - int firstEventData[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - EventBuilder builder(1, DataType::INT32, 2); - std::shared_ptr firstEv = builder.getEvent(); - try { - builder.setIntData(firstEv, firstEventData, 10); - } - catch (EvioException &e) {/* never happen */} - - bool append = false; - // Allow only 2 events per block to test using multiple blocks - int blockCount = 20000; - - try { - EventWriterV4 writer( - buffer, - EventWriterV4::DEFAULT_BLOCK_SIZE, - blockCount, - dictionary, nullptr, 0, - 1, append, firstEv); - - // Create an event in buffer form with lots of stuff in it - std::shared_ptr evioDataBuf = createCompactEventBuffer(tag, num, order); - // Create same event as EvioEvent object - std::shared_ptr evioEv = createEventBuilderEvent(tag, num); - // Create node from this buffer - std::shared_ptr node = EvioNode::extractEventNode(evioDataBuf, 0, 0, 0); - - - // write as buffer - writer.writeEvent(evioDataBuf, false); - // write as EvioEvent - writer.writeEvent(evioEv, false); - // write as node - writer.writeEvent(node, false); - - writer.close(); - // Get ready-to-read buffer - buffer = writer.getByteBuffer(); - - } - catch (EvioException &e) { - std::cout << "PROBLEM: " << e.what(); - } - - std::shared_ptr copy = ByteBuffer::copyBuffer(buffer); - std::shared_ptr copy2 = ByteBuffer::copyBuffer(buffer); - - - // Reader cannot be used, it only works on evio version 6 files and buffers. - // Reader reader(buffer); - - - - std::cout << "--------------------------------------------" << std::endl; - std::cout << "---------- EvioCompactReader -----------" << std::endl; - std::cout << "--------------------------------------------" << std::endl; - - std::shared_ptr dataBuf0 = nullptr; - - try { - EvioCompactReader reader1(copy); - - uint32_t evCount2 = reader1.getEventCount(); - std::cout << " Got " << evCount2 << " events" << std::endl; - - std::string dict2 = reader1.getDictionaryXML(); - std::cout << " Got dictionary = \n" << dict2 << std::endl; - - // Compact reader does not deal with first events, so skip over it - - std::cout << "\n Print out events (includes first event if evio version 4) :" << std::endl; - - for (uint32_t i = 0; i < evCount2; i++) { - // The "first event" is just the first event in the list (not treated specially) - std::cout << " scanned event #" << (i+1) << " :" << std::endl; - std::shared_ptr compactNode = reader1.getScannedEvent(i + 1); - std::cout << " node ->\n " << compactNode->toString() << std::endl; - - std::shared_ptr dataBuf = std::make_shared(compactNode->getTotalBytes()); - compactNode->getStructureBuffer(dataBuf, true); - - if (i == 0) { - dataBuf0 = dataBuf; - Util::printBytes(dataBuf, dataBuf->position(), dataBuf->remaining(), - " Event #" + std::to_string(i+1)); - } - } - } - catch (EvioException &e) { - std::cout << "PROBLEM: " << e.what(); - } - - - std::cout << std::endl; - std::cout << "--------------------------------------------" << std::endl; - std::cout << "---------- EvioReader --------------" << std::endl; - std::cout << "--------------------------------------------" << std::endl; - - - bool unchanged = true; - size_t index = 0; - std::vector dataVec; - std::vector dataVec0; - - try { - - // TODO: Something WRONG with the parsing!!! - EvioReader reader1(copy2); - - uint32_t evCount2 = reader1.getEventCount(); - std::cout << " Got " << evCount2 << " events" << std::endl; - - std::string dict2 = reader1.getDictionaryXML(); - std::cout << " Got dictionary = \n" << dict2 << std::endl; - - std::cout << "\n Got first event = " << reader1.hasFirstEvent() << std::endl; - - std::cout << "\n Print out events (includes first event if evio version 4) :" << std::endl; - - for (uint32_t i = 0; i < evCount2; i++) { - std::shared_ptr ev = reader1.parseEvent(i + 1); - std::cout << " got & parsed ev " << (i+1) << std::endl; - std::cout << " event ->\n" << ev->toString() << std::endl; - if (i == 0) { - dataVec0 = ev->getRawBytes(); - } - //std::shared_ptr evt = reader1.parseNextEvent(); - } -// This has the same output as above -// int j=0; -// std::shared_ptr evt; -// while ((evt = reader1.parseNextEvent()) != nullptr) { -// std::cout << " got & parsed ev " << (j+1) << std::endl; -// std::cout << " event ->\n" << evt->toString() << std::endl; -// if (j == 0) { -// dataVec0 = evt->getRawBytes(); -// } -// j++; -// } - - std::cout << " Comparing buffer data (lim = " << dataBuf0->limit() << ") with vector data (len = " << dataVec0.size() - << ")" << std::endl; - for (size_t i = 0; i < dataVec0.size(); i++) { - if ((/*data[i+8]*/ dataBuf0->array()[i + 8] != dataVec0[i]) && (i > 3)) { - unchanged = false; - index = i; - std::cout << " Compact reader different than EvioReader at byte #" << index << std::endl; - std::cout << showbase << hex << dataBuf0->array()[i + 8] << " changed to " << - dataVec0[i] << dec << std::endl; - break; - } - } - if (unchanged) { - std::cout << "First data EVENT same whether using EvioCompactReader or EvioReader!" << std::endl; - } - - } - catch (EvioException &e) { - std::cout << "PROBLEM: " << e.what(); - } - - - - } - - - }; - - -} - - - - -int main(int argc, char **argv) { - - - string filename_c = "./evioTest.c.evio"; - string filename_j = "./evioTest.java.evio"; - - evio::ReadWriteTest tester; - - tester.writeFile(filename_c, 1,1); - tester.readFile(filename_c); - -// std::ifstream file(filename_j); -// if (file) { -// tester.readFile(filename_j); -// } - - tester.writeAndReadBuffer(1,1); - - // Buffers ... - cout << endl << endl << "----------------------------------------" << endl << endl; - - return 0; -} - diff --git a/src/test/cpp/EvioTestHelper.h b/src/test/cpp/EvioTestHelper.h new file mode 100644 index 000000000..2c788b9dc --- /dev/null +++ b/src/test/cpp/EvioTestHelper.h @@ -0,0 +1,172 @@ +// +// Copyright 2024, Jefferson Science Associates, LLC. +// Subject to the terms in the LICENSE file found in the top-level directory. +// +// EPSCI Group +// Thomas Jefferson National Accelerator Facility +// 12000, Jefferson Ave, Newport News, VA 23606 +// (757)-269-7100 + +#ifndef EVIO_TEST_HELPER_H +#define EVIO_TEST_HELPER_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "eviocc.h" + + +using namespace evio; + + + class EvioTestHelper { + + protected: + + + public: + + // Constructor + explicit EvioTestHelper() {} + + std::shared_ptr defaultEventWriterV4() { + return std::make_shared( + baseNameV4, + directory, + runType, + runNumber, + split, + maxRecordSize, + maxEventCount, + byteOrder, + XMLdictionary, + overWriteOK, + append, + firstEvent, + streamId, + splitNumber, + splitIncrement, + streamCount, + bufferSize, + bitInfo + ); + } + + std::shared_ptr defaultEventWriter(std::string baseName = "") { + + if (baseName.empty()) { + baseName = baseNameV6; // Default to V6 if no base name is provided + } + + return std::make_shared( + baseName, + directory, + runType, + runNumber, + split, + maxRecordSize, + maxEventCount, + byteOrder, + XMLdictionary, + overWriteOK, + append, + firstEvent, + streamId, + splitNumber, + splitIncrement, + streamCount, + compressionType, + compressionThreads, + ringSize, + bufferSize + ); + } + std::shared_ptr defaultEventWriterHIPO() { + return std::make_shared( + baseNameHIPO, + directory, + runType, + runNumber, + split, + maxRecordSize, + maxEventCount, + byteOrder, + XMLdictionary, + overWriteOK, + append, + firstEvent, + streamId, + splitNumber, + splitIncrement, + streamCount, + compressionTypeHIPO, + compressionThreads, + ringSize, + bufferSize + ); + } + + std::vector genXYZT(int i) { + std::vector x4(4); // 5th entry for pyevio bugfix + x4[0] = gauss(gen); + x4[1] = gauss(gen); + x4[2] = 0.0f; + x4[3] = i*2.008f; + return x4; + } + std::string baseNameV4 = "testEventsV4_cppAPI.evio"; // base name of file to be created. If split > 1, this is the base name of all files created. If split < 1, this is the name of the only file created. + std::string baseNameV6 = "testEventsV6_cppAPI.evio"; // base name of file to be created. If split > 1, this is the base name of all files created. If split < 1, this is the name of the only file created. + std::string baseNameHIPO = "testEventsHIPO_cppAPI.hipo"; // base name of file to be created. If split > 1, this is the base name of all files created. If split < 1, this is the name of the only file created. + std::string directory = "./"; // directory in which file is to be placed + + private: + + // Random number generator for Gaussian (mean=0, sigma=0.1) + std::random_device my_rd; + std::mt19937 gen{ my_rd() }; + std::normal_distribution gauss{0.0f, 0.1f}; + + + const std::string runType = ""; // name of run type configuration to be used in naming files + uint32_t runNumber = 1; + uint64_t split = 0; // if < 1, do not split file, write to only one file of unlimited size. Else this is max size in bytes to make a file before closing it and starting writing another. + uint32_t maxRecordSize = 33554432; // (32 MiB) max number of uncompressed data bytes each record can hold. Value of < 8MB results in default of 8MB. The size of the record will not be larger than this size unless a single event itself is larger + uint32_t maxEventCount = 10000; // max number of events each record can hold. Value <= O means use default (1M). + const ByteOrder & byteOrder = ByteOrder::nativeOrder(); + const std::string XMLdictionary = R"( + + + + + + + + + + )"; + // + // + bool overWriteOK = true; + bool append = false; + std::shared_ptr< EvioBank >firstEvent = nullptr; // The first event written into each file (after any dictionary) including all split files; may be null. Useful for adding common, static info into each split file + uint32_t streamId = 1; // streamId number (100 > id > -1) for file name + uint32_t splitNumber = 0; // number at which to start the split numbers + uint32_t splitIncrement = 1; // amount to increment split number each time another file is created + uint32_t streamCount = 1; // total number of streams in DAQ + Compressor::CompressionType compressionType = Compressor::UNCOMPRESSED; + Compressor::CompressionType compressionTypeHIPO = Compressor::LZ4; + uint32_t compressionThreads = 1; // number of threads doing compression simultaneously + uint32_t ringSize = 0; // number of records in supply ring. If set to < compressionThreads, it is forced to equal that value and is also forced to be a multiple of 2, rounded up. + size_t bufferSize = 33554432; // (32 MiB) number of bytes to make each internal buffer which will be storing events before writing them to a file. 9MB = default if bufferSize = 0. + std::bitset<24>* bitInfo = nullptr; // EventWriterV4 only, set of bits to include in first block header + }; + +#endif //EVIO_TEST_HELPER_H diff --git a/src/test/cpp/EvioWriteAndReadBack_builder.cpp b/src/test/cpp/EvioWriteAndReadBack_builder.cpp new file mode 100644 index 000000000..ae1f34341 --- /dev/null +++ b/src/test/cpp/EvioWriteAndReadBack_builder.cpp @@ -0,0 +1,167 @@ +#include "EvioTestHelper.h" // EVIO C++ API (EVIO 6 main branch) + +using namespace evio; + +// Forward declaration for a few functions +void evioReadStep(std::string filename); + +int main(int argc, char* argv[]) { + + // Boilerplate + if (argc != 2) { + std::cerr << "Usage: " << argv[0] << " \n"; + return 1; + } + int nEvents = std::stoi(argv[1]); + std::cout << "Writing " << nEvents << " events to files..." << std::endl; + EvioTestHelper* evioHelperObj = new EvioTestHelper(); + + // std::shared_ptr writerV4 = evioHelperObj->defaultEventWriterV4(); + std::shared_ptr writerV6 = evioHelperObj->defaultEventWriter(); + // std::shared_ptr writerHipo = evioHelperObj->defaultEventWriterHIPO(); + + for (int i = 0; i < nEvents; ++i) { + + // Build a new event (top-level bank) with tag=1, type=BANK, num=1 + //------------------------------------- + uint16_t tag = 1; + uint8_t num = 1; + // if(i == 0) { + std::shared_ptr builder = std::make_shared(tag, DataType::BANK, num); + std::shared_ptr event = builder->getEvent(); + // } + // else { + // builder = std::make_shared(event); + // } + + std::vector floatVec = evioHelperObj->genXYZT(i); // generate pseudo x, y, z, time values + // std::vector floatVec = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f}; // Example data for testing + + // Now to start defining event + + // // THE OVERBANK + // // First child of event = bank of banks + // auto bankBanks = EvioBank::getInstance(tag+1, DataType::BANK, num+1); + // builder->addChild(event, bankBanks); + + // // (SUB)BANK 1 OF 1 + // // Create first (& only) child of bank of banks = bank of floats + std::shared_ptr bankFloats = EvioBank::getInstance(tag+11, DataType::FLOAT32, num+11); + // // Write our data into bank + builder->setFloatData(bankFloats, floatVec.data(), floatVec.size()); + builder->addChild(event, bankFloats); + + // Write the completed event to file + writerV6->writeEvent(event); + + // event.reset(); + // bankFloats.reset(); + // builder.reset(); + // floatVec.clear(); + } + + + writerV6->close(); // close file writer (flush remaining data)​:contentReference[oaicite:6]{index=6} + std::cout << "Wrote " << nEvents << " events to file." << std::endl; + + evioReadStep(evioHelperObj->directory + "/" + evioHelperObj->baseNameV6); + + return 0; +} + + +void evioReadStep(std::string filename) { + + EvioReader reader(filename); + + // Find out how many events the file contains + int32_t evCount = reader.getEventCount(); + int32_t eventsToCheck = 10; // Number of events to check with explicit output + std::cout << "File has " << evCount << " events" << std::endl; + + // Does it contain a dictionary? + if (reader.hasDictionaryXML()) { + std::string dict = reader.getDictionaryXML(); + std::cout << "Dictionary = " << dict << std::endl; + } + + // Does it contain a first event? + std::shared_ptr fe = reader.getFirstEvent(); + if (fe != nullptr) std::cout << "First event size = " << fe->getTotalBytes() << " bytes" << std::endl; + + std::cout << "EVIO Ver: " << reader.getEvioVersion() << std::endl; + std::cout << "File Size: " << reader.fileSize() << std::endl; + std::cout << "Event count: " << reader.getEventCount() << std::endl; + std::cout << "Has first event?: " << reader.hasFirstEvent() << std::endl; + + // Look at each event with random-access type method (start at 1) + std::cout << "Print out regular events raw data:" << std::endl; + for (int i = 0; i < evCount; i++) { + + if (i >= eventsToCheck) { + std::cout << "Reached maximum number of events to read: " << eventsToCheck << std::endl; + break; + } + + auto ev = reader.parseEvent(i+1); + std::cout << " got & parsed ev " << (i+1) << std::endl; + std::cout << " event ->\n" << ev->toString() << std::endl; + auto& dataVec = ev->getRawBytes(); + std::cout << "Event has tag = " << ev->getHeader()->getTag() << std::endl; + std::cout << "Event structure type = " << ev->getStructureType().toString() << std::endl; + std::vector< std::shared_ptr< BaseStructure > > & children = ev->getChildren(); + std::cout << "Event has " << children.size() << " children" << std::endl; + + // Now loop over children, if any + for (size_t j = 0; j < children.size(); j++) { + std::cout << "Child " << j << " tag = " << children[j]->getStructureType().toString() << std::endl; + + std::cout << "NChildren: " << children[j]->getChildCount() << std::endl; + std::cout << "Num items stored: " << children[j]->getNumberDataItems() << std::endl; + std::cout << "Data type: " << children[j]->getHeader()->getDataType().toString() << std::endl; + if(children[j]->getHeader()->getDataType() == DataType::FLOAT32) { + std::cout << "Data: "; + std::vector data_vec = children[j]->getFloatData(); + for (size_t l = 0; l < data_vec.size(); l++) { + std::cout << data_vec[l] << " "; + } + + // for (size_t k = 0; k < children[j]->getChildCount(); k++) { + // std::cout << "Child " << j << ", subchild " << k << " " << children[j]->getChildAt(k)->getStructureType().toString() << std::endl; + // std::cout << "Subchild " << k << " datatype: " << children[j]->getChildAt(k)->getHeader()->getDataType().toString() << std::endl; + // std::cout << "nsubchildren: " << children[j]->getChildAt(k)->getChildCount() << std::endl; + // std::cout << "num items stored: " << children[j]->getChildAt(k)->getNumberDataItems() << std::endl; + // // If data type is int, dump the data + // std::cout << "Data type: " << children[j]->getChildAt(k)->getHeader()->getDataType().toString() << std::endl; + // if(children[j]->getChildAt(k)->getHeader()->getDataType() == DataType::FLOAT32) { + // std::cout << "Data: "; + // std::vector data_uint_vec = children[j]->getChildAt(k)->getFloatData(); + // for (size_t l = 0; l < data_uint_vec.size(); l++) { + // std::cout << data_uint_vec[l] << " "; + // } + // std::cout << std::endl; + // } + // } + } + std::cout << std::endl; + + + reader.rewind(); + + } + } + + return; +} + + // Go back to beginning of file + + // // Use sequential access to events + // std::shared_ptr ev = nullptr; + // int counter = 0; + // while ((ev = reader.parseNextEvent()) != nullptr) { + // counter++; + // if (counter >= eventsToCheck) break; + // // do something with event + // std::cout << "Event has tag = " << ev->getHeader()->getTag() << std::endl; + // } diff --git a/src/test/cpp/EvioWriteAndReadBack_compact.cpp b/src/test/cpp/EvioWriteAndReadBack_compact.cpp new file mode 100644 index 000000000..f74ce21d5 --- /dev/null +++ b/src/test/cpp/EvioWriteAndReadBack_compact.cpp @@ -0,0 +1,71 @@ +#include "EvioTestHelper.h" + +using namespace evio; + +int main(int argc, char** argv) { + + // Boilerplate + if (argc != 2) { + std::cerr << "Usage: " << argv[0] << " \n"; + return 1; + } + int nEvents = std::stoi(argv[1]); + std::cout << "Writing " << nEvents << " events to files..." << std::endl; + EvioTestHelper* evioHelperObj = new EvioTestHelper(); + + // std::shared_ptr writerV4 = evioHelperObj->defaultEventWriterV4(); + std::shared_ptr writerV6 = evioHelperObj->defaultEventWriter(); + // std::shared_ptr writerHipo = evioHelperObj->defaultEventWriterHIPO(); + + for (int i = 0; i < nEvents; ++i) { + + + // Data to write stored in these arrays + // int8_t byteArray[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + std::vector floatArr = evioHelperObj->genXYZT(i); // generate pseudo x, y, z, time values; + // Create the buffer to hold everything + ByteBuffer buffer = ByteBuffer(8192); + // Build event (bank of banks) with CompactEventBuilder object + int tag = 1, num = 1; + CompactEventBuilder* builder = new CompactEventBuilder(8192,ByteOrder::nativeOrder()); + //------------------------------------- + // add top/event level bank of banks + builder->openBank(tag, DataType::BANK, num); + // add bank of banks to event + builder->openBank(tag+1, DataType::BANK, num+1); + // add bank of ints to bank of banks + builder->openBank(tag+11, DataType::FLOAT32, num+11); + // done with bank of ints, go to enclosing structure + builder->addFloatData(floatArr.data(),floatArr.size()); + + builder->closeStructure(); + // done with bank of banks + builder->closeStructure(); + builder->closeAll(); + // There is no way to remove any structures + // Get ready-to-read buffer from builder + // (this sets the proper pos and lim in buffer) + std::shared_ptr& bytebbuffer = builder->getBuffer(); + + // Create an event as a bank containing four 32-bit floats + std::shared_ptr event = EvioBank::getInstance(1, DataType::FLOAT32, 1); + // Fill the bank's float data (x, y, z, time) with 0.0 + auto &floatData = event->getFloatData(); + floatData = evioHelperObj->genXYZT(i); // generate pseudo x, y, z, time values + event->updateFloatData(); // update internal length counters​:contentReference[oaicite:3]{index=3} + + // Write the event to each file format + // writerV4->writeEvent(event); + writerV6->writeEvent(event); + // writerHipo->writeEvent(event); + } + + // Close the writers, flush buffers + // writerV4->close(); + writerV6->close(); + // writerHipo->close(); + + std::cout << "Wrote " << nEvents << " events to EVIO4, EVIO6, and HIPO format files." << std::endl; + delete evioHelperObj; + return 0; +} diff --git a/src/test/cpp/EvioWriteAndReadBack_simple.cpp b/src/test/cpp/EvioWriteAndReadBack_simple.cpp new file mode 100644 index 000000000..093d765e8 --- /dev/null +++ b/src/test/cpp/EvioWriteAndReadBack_simple.cpp @@ -0,0 +1,42 @@ +#include "EvioTestHelper.h" // EVIO C++ API (EVIO 6 main branch) + +using namespace evio; + +int main(int argc, char** argv) { + + // Boilerplate + if (argc != 2) { + std::cerr << "Usage: " << argv[0] << " \n"; + return 1; + } + int nEvents = std::stoi(argv[1]); + std::cout << "Writing " << nEvents << " events to files..." << std::endl; + EvioTestHelper* evioHelperObj = new EvioTestHelper(); + + // std::shared_ptr writerV4 = evioHelperObj->defaultEventWriterV4(); + std::shared_ptr writerV6 = evioHelperObj->defaultEventWriter(); + // std::shared_ptr writerHipo = evioHelperObj->defaultEventWriterHIPO(); + + for (int i = 0; i < nEvents; ++i) { + // Create an event as a bank containing four 32-bit floats + std::shared_ptr event = EvioBank::getInstance(1, DataType::FLOAT32, 1); + // Fill the bank's float data (x, y, z, time) with 0.0 + auto &floatData = event->getFloatData(); + floatData = evioHelperObj->genXYZT(i); // generate pseudo x, y, z, time values + event->updateFloatData(); // update internal length counters​:contentReference[oaicite:3]{index=3} + + // Write the event to each file format + // writerV4->writeEvent(event); + writerV6->writeEvent(event); + // writerHipo->writeEvent(event); + } + + // Close the writers, flush buffers + // writerV4->close(); + writerV6->close(); + // writerHipo->close(); + + std::cout << "Wrote " << nEvents << " events to EVIO4, EVIO6, and HIPO format files." << std::endl; + delete evioHelperObj; + return 0; +} diff --git a/src/test/java/EvioTestHelper.java b/src/test/java/EvioTestHelper.java new file mode 100644 index 000000000..26dde7a5b --- /dev/null +++ b/src/test/java/EvioTestHelper.java @@ -0,0 +1,135 @@ + +import java.util.Random; +import java.nio.ByteOrder; + +import org.jlab.coda.hipo.CompressionType; +// Import evio +import org.jlab.coda.jevio.*; + +public class EvioTestHelper { + private final Random random; + + // Parameters (simplified here for brevity) + public static String baseNameV4 = "testEventsV4_javaAPI.evio"; // base name of file to be created. If split > 1, this is the base name of all files created. If split < 1, this is the name of the only file created. + public static String baseNameV6 = "testEventsV6_javaAPI.evio"; // base name of file to be created. If split > 1, this is the base name of all files created. If split < 1, this is the name of the only file created. + public static String baseNameHIPO = "testEventsHIPO_javaAPI.hipo"; // base name of file to be created. If split > 1, this is the base name of all files created. If split < 1, this is the name of the only file created. + public static String directory = "tmp"; // "/mnt/tmpfs" or null=relative path (or directory in which file is to be placed) + final static String runType = ""; // name of run type configuration to be used in naming files + private final static int runNumber = 1; // arbitrary, usually experiment-specific + private final static long split = 0; // if < 1, do not split file, write to only one file of unlimited size. Else this is max size in bytes to make a file before closing it and starting writing another. + private final static int maxRecordSize = 33554432; // (32 MiB) max number of uncompressed data bytes each record can hold. Value of < 8MB results in default of 8MB. The size of the record will not be larger than this size unless a single event itself is larger + private final static int maxEventCount = 10000; // max number of events each RECORD can hold. Value <= O means use default (1M). + private final static ByteOrder byteOrder = ByteOrder.nativeOrder(); // options: BIG_ENDIAN, LITTLE_ENDIAN, nativeOrder() + private final static String XMLdictionary = + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + // " \n" + + // " \n" + + "\n"; + private final static boolean overWriteOK = true; + private final static boolean append = false; + private final static EvioBank firstEvent = null; // The first event written into each file (after any dictionary) including all split files; may be null. Useful for adding common, static info into each split file + private final static int streamId = 1; // streamId number (100 > id > -1) for file name + private final static int splitNumber = 0; // number at which to start the split numbers + private final static int splitIncrement = 1; // amount to increment split number each time another file is created + private final static int streamCount = 1; // total number of streams in DAQ + private final static CompressionType compressionType = CompressionType.RECORD_UNCOMPRESSED; + private final static CompressionType compressionTypeHIPO = CompressionType.RECORD_COMPRESSION_LZ4; + private final static int compressionThreads = 1; // number of threads doing compression simultaneously + private final static int ringSize = 0; // number of records in supply ring. If set to < compressionThreads, it is forced to equal that value and is also forced to be a multiple of 2, rounded up. + private final static int bufferSize = 33554432; // (32 MiB) number of bytes to make each internal buffer which will be storing events before writing them to a file. 9MB = default if bufferSize = 0. + + // Constructor + public EvioTestHelper() { + this.random = new Random(); + } + + // Generate a float array similar to C++ vector + public float[] genXYZT(int i) { + float[] x4 = new float[4]; + x4[0] = (float) random.nextGaussian() * 0.1f; + x4[1] = (float) random.nextGaussian() * 0.1f; + x4[2] = 0.0f; + x4[3] = i * 2.008f; + return x4; + } + + public static EventWriterV4 defaultEventWriterV4() { + + // EventWriterV4 w = null; + + EventWriterV4 w = null; + + try { + + w = new EventWriterV4( + baseNameV4, + directory, + runType, + runNumber, + split, + maxRecordSize, + maxEventCount, + 10000, + byteOrder, + XMLdictionary, + null, + overWriteOK, + append, + firstEvent, + streamId, + splitNumber, + splitIncrement, + streamCount +); + } + catch (EvioException e) { + System.err.println("Error creating writer. Stack trace:\n "); + e.printStackTrace(); + } + return w; + } + + // Default writer + public static EventWriter defaultEventWriter() { + + EventWriter w = null; + + try { + w = new EventWriter( + baseNameV6, + directory, + runType, + runNumber, + split, + maxRecordSize, + maxEventCount, + byteOrder, + XMLdictionary, + overWriteOK, + append, + firstEvent, + streamId, + splitNumber, + splitIncrement, + streamCount, + compressionType, + compressionThreads, + ringSize, + bufferSize); + } + catch (EvioException e) { + System.err.println("Error creating writer. Stack trace:\n "); + e.printStackTrace(); + } + return w; + } + + +} diff --git a/src/test/java/EvioWriteAndReadBack_builder.java b/src/test/java/EvioWriteAndReadBack_builder.java new file mode 100644 index 000000000..1be3af390 --- /dev/null +++ b/src/test/java/EvioWriteAndReadBack_builder.java @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2025. Jefferson Lab (JLab). All rights reserved. Permission + * to use, copy, modify, and distribute this software and its documentation for + * educational, research, and not-for-profit purposes, without fee and without a + * signed licensing agreement. + */ + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; + +import java.io.*; +import org.jlab.coda.jevio.*; + +@Tag("fast") // Run when selecting "fast" category of tests + +class EvioTestReadback_builder { + + @Test + @DisplayName("Evio Write and Read Back Test") + void DoReadbackTest(TestInfo testInfo) throws IOException, EvioException { + + int nEvents = 100000; // Number of events to write + + evioWriteStep(nEvents); // Write events + evioReadStep(nEvents); // Readback with asserts + } + + + void evioWriteStep(int nEvents) throws IOException, EvioException { + + EvioTestHelper h = new EvioTestHelper(); + + // Create a new EvioWriter + EventWriterUnsync writer = EvioTestHelper.defaultEventWriter(); + + EventBuilder builder = null; + EvioEvent event = null; + + // Generate and write events + for (int i = 0; i < nEvents; i++) { + + // Build a new event (top-level bank) with tag=1, type=BANK, num=1 + //------------------------------------- + int tag = 1; + int num = 1; + + float[] floatVec = h.genXYZT(i); // generate pseudo x, y, z, time values + + builder = new EventBuilder(tag, DataType.BANK, num); + event = builder.getEvent(); + + // (SUB)BANK 1 OF 1 + // Create first (& only) child of bank of banks = bank of floats + EvioBank bankFloats = new EvioBank(tag+11, DataType.FLOAT32, num+11); + bankFloats.appendFloatData(floatVec); // add float data to bank of floats + builder.addChild(event, bankFloats); + + try { + // Write the completed event to file + writer.writeEvent(event, false); + } + catch (EvioException e) { + System.out.println("EvioException: " + e.toString()); + } + } + // End of writing events + try { + writer.close(); + } + catch (Exception e) { + System.out.println("EvioException: " + e.toString()); + e.printStackTrace(); + } + + System.out.println(" Wrote " + nEvents + " events to file. "); + + } + + + void evioReadStep(int nEvents) throws IOException, EvioException { + + String filename = EvioTestHelper.directory + "/" + EvioTestHelper.baseNameV6; + + try { + EvioReader reader = new EvioReader(filename); + int evCount = reader.getEventCount(); + int eventsToCheck = 10; // Number of events to check with explicit output + + System.out.println("Read " + evCount + " events from file: " + filename); + assertEquals(nEvents, evCount, "Event count mismatch after reading back"); + + // Does it contain a dictionary? + // if (reader.hasDictionaryXML()) { + // String dict = reader.getDictionaryXML(); + // System.out.println("Dictionary = \n" + dict); + // } + + // Look at each event with random-access type method (starts at 1) + // System.out.println("Print out regular events’ raw data:"); + // for (int i = 0; i < eventsToCheck; i++) { + // EvioEvent ev = reader.parseEvent(i + 1); + // byte[] byteData = ev.getByteData(); + // System.out.println("First ev size = " + ev.getTotalBytes() + " bytes"); + // System.out.println(" Event # " + i + " length = " + byteData.length + " ints"); + // Utilities.printBytes(byteData, 0, byteData.length, " Event #" + i); + // } + + reader.rewind(); // Rewind to the beginning of the file + // for (int i = 0; i < eventsToCheck; i++) { + // EvioEvent ev = reader.parseNextEvent(); + // byte[] byteData = ev.getByteData(); + // System.out.println("First ev size = " + ev.getTotalBytes() + " bytes"); + // // System.out.println(" Event # " + i + " length = " + byteData.length + " ints"); + // Utilities.printBytes(byteData, 0, byteData.length, " Event #" + i); + // } + + + + // reader.rewind(); // Rewind to the beginning of the file + // // Look at each event with sequential type method (starts at 0) + // for (int i = 0; i < evCount; i++) { + // EvioEvent ev = reader.parseNextEvent(); + // byte[] byteData = ev.getByteData(); + // Utilities.printBytes(byteData, 0, byteData.length, " Event #" + i); + + // // Check if the event is a bank + // if (ev.getStructureType() == StructureType.BANK) { + // List banks = ev.getChildBanks(); + // assertEquals(1, banks.size(), "Expected 1 child bank in event " + i); + // // Look at each event with sequential type method (starts at 0) + // for (int i = 0; i < evCount; i++) { + // EvioEvent ev = reader.parseNextEvent(); + // byte[] byteData = ev.getByteData(); + // Utilities.printBytes(byteData, 0, byteData.length, " Event #" + i); + + // // Check if the event is a bank + // if (ev.getStructureType() == StructureType.BANK) { + // List banks = ev.getChildBanks(); + // assertEquals(1, banks.size(), "Expected 1 child bank in event " + i); + + // EvioBank bank = banks.get(0); + // assertEquals(1, bank.getNum(), "Expected num=1 for bank in event " + i); + + // // Check the float data + // float[] data = bank.getFloatData(); + // assertEquals(4, data.length, "Expected 4 float values in bank of event " + i); + // // Check the float data + // float[] data = bank.getFloatData(); + // assertEquals(4, data.length, "Expected 4 float values in bank of event " + i); + + // // Validate the data + // // float[] expectedData = EvioTestHelper.genXYZT(i); + // assertEquals(expectedData[0], data[0], 0.001, "X value mismatch in event " + i); + // assertEquals(expectedData[1], data[1], 0.001, "Y value mismatch in event " + i); + // assertEquals(expectedData[2], data[2], 0.001, "Z value mismatch in event " + i); + // assertEquals(expectedData[3], data[3], 0.001, "T value mismatch in event " + i); + // } + // } + // } + // // Validate the data + // float[] expectedData = EvioTestHelper.genXYZT(i); + // assertEquals(expectedData[0], data[0], 0.001, "X value mismatch in event " + i); + // assertEquals(expectedData[1], data[1], 0.001, "Y value mismatch in event " + i); + // assertEquals(expectedData[2], data[2], 0.001, "Z value mismatch in event " + i); + // assertEquals(expectedData[3], data[3], 0.001, "T value mismatch in event " + i); + // } + // } + } + catch (Exception e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/tar/tarexclude b/tar/tarexclude deleted file mode 100644 index 138c1844e..000000000 --- a/tar/tarexclude +++ /dev/null @@ -1,16 +0,0 @@ -.git -.svn -.scon* -*~ -coda.pyc -*.iml -config.log -doxyerrors.log -doc/doxygen/* -tar/evio* -src/libsrc/.* -src/libsrc++/.* -src/execsrc/.* -src/examples/.* -src/test/.* -build diff --git a/tmp/README.md b/tmp/README.md new file mode 100644 index 000000000..71d089bc7 --- /dev/null +++ b/tmp/README.md @@ -0,0 +1,86 @@ + +Here is how to convert the CLAS12 pcap files into evio and then into ROOT trees. + +Note that a single pcap file has packets destined for 24 different ports. Each stream maintains its own EVIO format so the streams must be split and then the payloads extracted in order to get valid EVIO. + +## Get the pcap headers and libraries + +This is a round-about way of getting these on the ifarm. If the libpcap-devel package were installed there, then this would not be needed. It is a quick and dirty recipe though for getting these by installing the package in a docker container that is using the same almalinux 9.5 OS that the ifarm is currently using. The files are just copied from there. Skip this if you already have them through other means. +~~~bash +mkdir -p opt/include opt/lib64 +podman run -it --rm -v $PWD/opt:/work docker://almalinux:9.5 bash + +dnf install -y 'dnf-command(config-manager)' +dnf config-manager --set-enabled crb +dnf install -y libpcap-devel + +cp -rp /usr/include/*pcap* /work/include +cp -rp /usr/lib64/*pcap* /work/lib64 +exit +~~~ + +## Build the first two tools. + +Clone the existing SRO-RTDP repository and build the pcapSplit and pcpa2evio tools. + +~~~bash +git clone git@github.com:JeffersonLab/SRO-RTDP + +g++ -g -std=c++2a -o pcapSplit -Iopt/include -Lopt/lib64 SRO-RTDP/src/utilities/cpp/pcapSplit/pcapSplit.cc -lpcap +g++ -g -std=c++2a -o pcap2evio -Iopt/include -Lopt/lib64 SRO-RTDP/src/utilities/cpp/pcap2evio/pcap2evio.cc -lpcap +~~~ + +## Convert into EVIO +Here are commands to split the captured *.pcap* file and then convert one of them into *evio*. You will only need to run the split command once as that should create 24 *pcap* files. One for each stream. The *pcap2evio* command will need to be run on each single split file you want to process. Thus, if you process all of them, you will end up with 24 *evio* files. +~~~bash +jcache get /mss/epsci/RTDP/2024.05.16.RTDP_CLAS12/CLAS12_ECAL_PCAL_DC_2024-05-16_09-07-18.pcap +ln -s /cache/epsci/RTDP/2024.05.16.RTDP_CLAS12/CLAS12_ECAL_PCAL_DC_2024-05-16_09-07-18.pcap +./pcapSplit CLAS12_ECAL_PCAL_DC_2024-05-16_09-07-18.pcap + +# Repeat this for each port +./pcap2evio CLAS12_ECAL_PCAL_DC_2024-05-16_09-07-18.pcap_split/port7001.pcap +~~~ + +## Convert the EVIO file(s) into *ROOT* +This is just an example of reading an *evio* created in the previous step and plotting one of the braches. + +n.b. This does minimal EVIO parsing and does *not* use the EVIO library. It should be easy to though, but it will need to use the EVIO-4 library routines. This is because the VTP puts out EVIO-4 so that is what was captured. +~~~bash +module use /cvmfs/oasis.opensciencegrid.org/jlab/scicomp/sw/el9/modulefiles +module load root/6.30.06-gcc11.4.0 +g++ -g -std=c++17 -o evio2root SRO-RTDP/src/utilities/cpp/evio2root/evio2root.cc `root-config --cflags --libs` + +# Repeat this for each port +./evio2root CLAS12_ECAL_PCAL_DC_2024-05-16_09-07-18.pcap_split/port7001.evio + +root -l CLAS12_ECAL_PCAL_DC_2024-05-16_09-07-18.pcap_split/port7001.evio.root +root [1] dcrb->Draw("t") +~~~ + +
+ +## Build EVIO + +This is an attempt to use the official EVIO library v6.0.1 to read the *evio* file(s) produced by the above. + +First, checkout and build the EVIO library +~~~bash +git clone git@github.com:JeffersonLab/evio +cd evio +git checkout v6.0.1 +cd - + +cmake -S evio -B build.evio -DCMAKE_INSTALL_PREFIX=${PWD}/opt -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_CXX_STANDARD=20 +cmake --build build.evio --target install -j 16 +~~~ + +## Build the evioReadTest program + +~~~bash +g++ -g -std=c++2a evioReadTest.cc -o evioReadTest -Iopt/include -Lopt/lib64 -Lopt/lib -levio -leviocc -Wl,-rpath,${PWD}/opt/lib64 +~~~ + + +~~~bash +./evioReadTest CLAS12_ECAL_PCAL_DC_2024-05-16_09-07-18.pcap_split/port7001.evio +~~~ \ No newline at end of file