From 3d6df89aeffe3e8769285fd42160de47e4aa8786 Mon Sep 17 00:00:00 2001 From: jonzarling Date: Fri, 25 Jul 2025 14:12:14 -0500 Subject: [PATCH 01/14] Add install to README instructions (cmake C++ build). Add new utilities folder with first skeleton program to merge evio files. --- .gitignore | 2 + CMakeLists.txt | 27 ++++++- README.md | 4 +- src/utils/cpp/evio_merge_files.cpp | 117 +++++++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 src/utils/cpp/evio_merge_files.cpp diff --git a/.gitignore b/.gitignore index 8457866f8..79dc7dccf 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,5 @@ doc/doxygen/C doc/source/ config.log +# Ignore typical install directory type +Linux-*/ diff --git a/CMakeLists.txt b/CMakeLists.txt index c2aa58b5e..f60244d27 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,7 @@ elseif (DEFINED ENV{CODA}) else() # Use default CMAKE_INSTALL_PREFIX set(INSTALL_DIR_DEFINED 1) + set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_SOURCE_DIR}/${ARCH}) message(STATUS "Installing to default location: ${CMAKE_INSTALL_PREFIX}") endif() @@ -98,12 +99,13 @@ file(GLOB C_LIB_FILES "src/libsrc/*.c") # C++ source files file(GLOB CPP_LIB_FILES "src/libsrc++/*.cpp") file(GLOB CPP_HEADER_FILES "src/libsrc++/*.h") -# A few extras required +# C++ utility files +file(GLOB CPP_UTILS_FILES "src/utils/cpp/*.cpp") +# A few extras required for examples if(MAKE_EXAMPLES) + list(APPEND CPP_HEADER_FILES src/test/cpp/EvioTestHelper.h) file(GLOB TEST "src/test/cpp/*.cpp") file(GLOB TESTC "src/test/c/*.c") - list(APPEND CPP_HEADER_FILES src/test/cpp/EvioTestHelper.h) - # list(APPEND CPP_LIB_FILES src/test/cpp/EvioTestHelper.h) endif() # BUILD C++ LIBRARY (unless otherwise specified) @@ -142,6 +144,25 @@ if(NOT C_ONLY) ${DISRUPTOR_INCLUDE_DIR} ) + # Build utility programs + foreach(fileName ${CPP_UTILS_FILES}) + # Get file name with no directory or extension as executable name + get_filename_component(execName ${fileName} NAME_WE) + # Create executable from file + add_executable(${execName} ${fileName}) + # Put debug extension on if applicable + set_target_properties(${execName} PROPERTIES DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX}) + # Needs these libs + target_link_libraries(${execName} eviocc pthread ${Boost_LIBRARIES} ${LZ4_LIBRARY} expat dl z m) + + # Only install if installation directory has been defined + if(DEFINED INSTALL_DIR_DEFINED) + message(STATUS "Installing utility executable: ${execName}") + # Install into bin/utils dir + install(TARGETS ${execName} RUNTIME DESTINATION bin/utils) + endif() + endforeach() + # Add the C++ tests/examples if(MAKE_EXAMPLES) foreach(fileName ${TEST}) diff --git a/README.md b/README.md index 5e3c0bb8f..6de5fcddc 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ To build C/C++ code from this repository: git clone https://github.com/JeffersonLab/evio/ cd evio; mkdir build cmake -S . -B build - cmake --build build --parallel + cmake --build build --target install --parallel Note that during the cmake configure step (first of two `cmake` commands above), one can toggle the following special flags: @@ -26,6 +26,8 @@ toggle the following special flags: * `MAKE_EXAMPLES`: build example/test programs (default `-DMAKE_EXAMPLES=0`) * `USE_FILESYSTEMLIB`: ue C++17 instead of Boost (default `-DUSE_FILESYSTEMLIB=0`) * `DISRUPTOR_FETCH`: allow CMake to download Disruptor if not found (default `-DDISRUPTOR_FETCH=1`) +* `CODA_INSTALL`: installs in this base directory. If not used, +then the env variable $CODA location is next checked. Otherwise defaults to \${CMAKE_HOST_SYSTEM_NAME}-\${CMAKE_HOST_SYSTEM_PROCESSOR}, typically something like `[evio_directory]/Linux-x86_64`. One can still also use `scons` instead of cmake to build the evio C/C++ library, though this feature will not be supported in future releases. diff --git a/src/utils/cpp/evio_merge_files.cpp b/src/utils/cpp/evio_merge_files.cpp new file mode 100644 index 000000000..ffdca1e93 --- /dev/null +++ b/src/utils/cpp/evio_merge_files.cpp @@ -0,0 +1,117 @@ +// +// Copyright 2025, Jefferson Science Associates, LLC. +// Subject to the terms in the LICENSE file found in the top-level directory. +// +// EPSCI Group +// Thomas Jefferson National Accelerator Facility +// 12000, Jefferson Ave, Newport News, VA 23606 +// (757)-269-7100 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "eviocc.h" + +using namespace evio; + +// Globals +static std::vector INFILENAMES; +static char* OUTFILENAME = nullptr; +static bool QUIT = false; + +// Prototypes +void ParseCommandLineArguments(int argc, char* argv[]); +void Usage(); +void ctrlCHandle(int); +void Process(unsigned int &NEvents, unsigned int &NEvents_read); +std::ifstream::pos_type GetFilesize(const char* filename); + +// ---------- GetFilesize ---------- +std::ifstream::pos_type GetFilesize(const char* filename) { + std::ifstream in(filename, std::ifstream::ate | std::ifstream::binary); + return in.tellg(); +} + +// ---------- main ---------- +int main(int argc, char* argv[]) { + signal(SIGINT, ctrlCHandle); + + ParseCommandLineArguments(argc, argv); + + // Print input files and their sizes + for (auto file : INFILENAMES) { + std::ifstream::pos_type size = GetFilesize(file); + std::cout << "Input file: " << file << " (size: " << size << " bytes)" << std::endl; + } + + unsigned int NEvents = 0; + unsigned int NEvents_read = 0; + Process(NEvents, NEvents_read); + + return 0; +} + +// ---------- ParseCommandLineArguments ---------- +void ParseCommandLineArguments(int argc, char* argv[]) { + INFILENAMES.clear(); + + for (int i = 1; i < argc; ++i) { + char* ptr = argv[i]; + if (ptr[0] == '-') { + switch (ptr[1]) { + case 'h': Usage(); break; + case 'o': OUTFILENAME = &ptr[2]; break; + default: + std::cerr << "Unknown option: " << ptr << std::endl; + Usage(); + } + } else { + INFILENAMES.push_back(ptr); + } + } + + if (INFILENAMES.empty()) { + std::cerr << "\nYou must specify at least one input file!\n" << std::endl; + Usage(); + } + + if (!OUTFILENAME) { + OUTFILENAME = new char[256]; + strcpy(OUTFILENAME, "merged.evio"); + } +} + +// ---------- Usage ---------- +void Usage() { + std::cout << "\nUsage:\n"; + std::cout << " evio_merge_files [-oOutputfile] file1.evio file2.evio ...\n\n"; + std::cout << "Options:\n"; + std::cout << " -oOutputfile Set output filename (default: merged.evio)\n"; + std::cout << "\nThis tool merges multiple EVIO files into one output file.\n"; + exit(0); +} + +// ---------- ctrlCHandle ---------- +void ctrlCHandle(int sig) { + QUIT = true; + std::cerr << "\nSIGINT received... exiting soon.\n"; +} + +// ---------- Process ---------- +void Process(unsigned int &NEvents, unsigned int &NEvents_read) { + std::cout << "Process() stub: would call into EVIO merge library here.\n"; + std::cout << "Output file: " << OUTFILENAME << std::endl; + std::cout << "Number of input files: " << INFILENAMES.size() << std::endl; + + // TODO: merge here + + +} From b7be09c5c71256a9787d6e4e633a79a1aae3473c Mon Sep 17 00:00:00 2001 From: jonzarling Date: Fri, 25 Jul 2025 15:17:27 -0500 Subject: [PATCH 02/14] Add actual code for merging program. Work on small test files at least. --- .gitignore | 1 + src/README.md | 3 +- src/utils/cpp/evio_merge_files.cpp | 62 ++++++++++++++++++++++++++---- 3 files changed, 58 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 79dc7dccf..d41e1ca6b 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ build/ jbuild/ lib/ bin/ +tmp/ doxyerrors.log target/ doc/javadoc diff --git a/src/README.md b/src/README.md index f99503b21..3c66dda7c 100644 --- a/src/README.md +++ b/src/README.md @@ -5,5 +5,6 @@ | [libsrc](libsrc) | EVIO C library | | [libsrc++](libsrc++) | EVIO C++ library | | [main/java](main/java) | EVIO Java Library | -| [test](test) | For internal testing of Java, C, C++ libraries. | +| [test](test) | For internal testing of Java, C, C++ libraries. | +| [utils](utils) | Standalone programs, e.g. utilities to merge files, convert formats, attempt recovery, etc. | diff --git a/src/utils/cpp/evio_merge_files.cpp b/src/utils/cpp/evio_merge_files.cpp index ffdca1e93..fb8d68fa7 100644 --- a/src/utils/cpp/evio_merge_files.cpp +++ b/src/utils/cpp/evio_merge_files.cpp @@ -106,12 +106,60 @@ void ctrlCHandle(int sig) { } // ---------- Process ---------- -void Process(unsigned int &NEvents, unsigned int &NEvents_read) { - std::cout << "Process() stub: would call into EVIO merge library here.\n"; - std::cout << "Output file: " << OUTFILENAME << std::endl; - std::cout << "Number of input files: " << INFILENAMES.size() << std::endl; +void Process(unsigned int &NEvents, unsigned int &NEvents_read) +{ + // Basic config + uint32_t maxRecordBytes = 1000000; + uint32_t maxEventsPerRecord = 1000; + size_t bufferBytes = 1000000; + std::string outFile(OUTFILENAME); + + // Use XML dict from first file, if any + std::string dictXml = ""; + try { + if (!INFILENAMES.empty()) { + EvioReaderV4 dictReader(INFILENAMES[0]); + if (dictReader.hasDictionaryXML()) { + dictXml = dictReader.getDictionaryXML(); + std::cout << "Dictionary found in first input file.\n"; + } else { + std::cout << "No dictionary found in first input file.\n"; + } + } + } catch (const std::exception &e) { + std::cerr << "Error retrieving dictionary from first file: " << e.what() << std::endl; + } - // TODO: merge here - + // Set up EVIO4 writer + std::unique_ptr writer = std::make_unique( + outFile, + "", "", 1, 0, + maxRecordBytes, maxEventsPerRecord, + ByteOrder::ENDIAN_LOCAL, + dictXml, + true, false, + nullptr, 1, 0, 1, 1, + bufferBytes + ); + + // Loop over all input files + for (auto filename : INFILENAMES) { + try { + std::cout << "Opening input file: " << filename << std::endl; + EvioReaderV4 reader(filename); + std::shared_ptr event; + + while ((event = reader.parseNextEvent())) { + writer->writeEvent(event); + NEvents_read++; + NEvents++; + } + } catch (const std::exception &e) { + std::cerr << "Error processing file " << filename << ": " << e.what() << std::endl; + continue; + } + } -} + writer->close(); + std::cout << "Done. " << NEvents_read << " events read, " << NEvents << " written." << std::endl; +} \ No newline at end of file From ce9772463231c00628a7e3caa593359d22ee53c7 Mon Sep 17 00:00:00 2001 From: Jon Zarling Date: Wed, 30 Jul 2025 16:59:56 -0500 Subject: [PATCH 03/14] Add java util for merging files, apparently more robust than analogous c++ util --- src/utils/java/convertFormats.java | 99 ++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 src/utils/java/convertFormats.java diff --git a/src/utils/java/convertFormats.java b/src/utils/java/convertFormats.java new file mode 100644 index 000000000..8bc86aa14 --- /dev/null +++ b/src/utils/java/convertFormats.java @@ -0,0 +1,99 @@ +import org.jlab.coda.jevio.*; +import org.jlab.coda.hipo.*; +import java.io.File; +import java.io.IOException; +import java.nio.ByteOrder; + +public class convertFormats { + public static void main(String[] args) throws IOException, EvioException { + // Parse command-line arguments + if (args.length < 1) { + System.err.println("You must specify at least one input file!"); + printUsage(); + return; + } + String outputFile = "merged.evio"; // default output name + java.util.List inputFiles = new java.util.ArrayList<>(); + for (String arg : args) { + if (arg.startsWith("-h")) { + printUsage(); + return; + } else if (arg.startsWith("-o")) { + // If option is "-oOutputName", set output file name + outputFile = arg.substring(2); + System.out.println("Output file: " + outputFile); + // prepend with "./" if directory not specified + if (!outputFile.startsWith("./") && !outputFile.startsWith("/")) { + // System.out.println("Using current directory"); + outputFile = "./" + outputFile; + } + + } else { + inputFiles.add(arg); + } + } + if (inputFiles.isEmpty()) { + System.err.println("No input files specified."); + printUsage(); + return; + } + + // Print input files and their sizes + for (String infile : inputFiles) { + long sizeBytes = new File(infile).length(); + System.out.println("Input file: " + infile + " (size: " + sizeBytes + " bytes)"); + } + + // Retrieve dictionary from the first file (if any) + String dictionaryXML = ""; + EvioReaderV4 dictReader = new EvioReaderV4(inputFiles.get(0)); + String xml = dictReader.getDictionaryXML(); // get dictionary if present:contentReference[oaicite:24]{index=24} + if (xml != null) { + dictionaryXML = xml; + System.out.println("Dictionary found in first input file."); + } + else System.out.println("No dictionary found in first input file."); + dictReader.close(); // close dictionary reader + + // Set up EVIO EventWriter for the merged output + int maxRecordBytes = 1000000; + int maxEventsPerRecord = 1000; + int bufferBytes = 1000000; + EventWriterV4 writer = new EventWriterV4( + outputFile, null, "", 1, 0L, + maxRecordBytes, maxEventsPerRecord, + bufferBytes, + ByteOrder.nativeOrder(), null, + null, true, false, + null, 0, 1, 1, 1 + ); + + // Process each input file: read all events and write to output + int eventsRead = 0; + int eventsWritten = 0; + for (String filename : inputFiles) { + EvioReaderV4 reader = new EvioReaderV4(filename); + System.out.println("Opening input file: " + filename); + EvioEvent event; + while ((event = reader.parseNextEvent()) != null) { // sequentially read events:contentReference[oaicite:25]{index=25} + // System.out.printf("Read event %d from %s%n", event.getEventNumber(), filename); + writer.writeEvent(event); // write event to output:contentReference[oaicite:26]{index=26} + eventsRead++; + eventsWritten++; + } + reader.close(); // close reader for this file + } + + // Close writer and report + writer.close(); // finalize output file:contentReference[oaicite:27]{index=27} + System.out.println("Done. " + eventsRead + " events read, " + eventsWritten + " written."); + } + + private static void printUsage() { + System.out.println("\nUsage:"); + System.out.println(" java EvioMergeFiles [-oOutputfile] file1.evio file2.evio ...\n"); + System.out.println("Options:"); + System.out.println(" -oOutputfile Set output filename (default: merged.evio)\n"); + System.out.println("This tool merges multiple EVIO files into one output file."); + } +} From a9318843b611cf8f595600792e856a93803cd676 Mon Sep 17 00:00:00 2001 From: Jon Zarling Date: Thu, 31 Jul 2025 11:35:16 -0500 Subject: [PATCH 04/14] Move links to top of readme again --- README.md | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 6de5fcddc..2c1e5e8eb 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,23 @@ within either a C/C++ or Java programming environment. # **Getting Started** +----------------------------- +## **Useful Links** +---------------------------- + +Documentation on GitHub: + +* [All Links](https://jeffersonlab.github.io/evio) +* [User's Guide PDF](https://jeffersonlab.github.io/evio/doc-6.0/users_guide/evio_Users_Guide.pdf) +* [EVIO Data Format Reference](https://jeffersonlab.github.io/evio/doc-6.0/format_guide/evio_Formats.pdf) + +Software Library Documentation: + +* [Javadoc for Java Library](https://jeffersonlab.github.io/evio/doc-6.0/javadoc/index.html) +* [Doxygen for C Library](https://jeffersonlab.github.io/evio/doc-6.0/doxygen/C/html/index.html) +* [Doxygen for C++ Libary](https://jeffersonlab.github.io/evio/doc-6.0/doxygen/CC/html/index.html) + + ## **C/C++ Library** To build C/C++ code from this repository: @@ -59,23 +76,10 @@ Requires Maven (`mvn`) and an installation of Java on your system. **Running on "ifarm" at JLab will not work unless you install java yourself**. Note that the default java versions on the farm will be too old to work. See downloads from [OpenJDK](https://openjdk.org/install/) or [Oracle](https://www.oracle.com/java/technologies/javase/jdk17-archive-downloads.html). ------------------------------ - -# **Useful Links** ---------------------------- - -Documentation on GitHub: - -* [All Documentation](https://jeffersonlab.github.io/evio) -* [User's Guide PDF](https://jeffersonlab.github.io/evio/doc-6.0/users_guide/evio_Users_Guide.pdf) -* [EVIO Data Format Reference](https://jeffersonlab.github.io/evio/doc-6.0/format_guide/evio_Formats.pdf) - -Software Library Documentation: - -* [Javadoc for Java Library](https://jeffersonlab.github.io/evio/doc-6.0/javadoc/index.html) -* [Doxygen for C Library](https://jeffersonlab.github.io/evio/doc-6.0/doxygen/C/html/index.html) -* [Doxygen for C++ Libary](https://jeffersonlab.github.io/evio/doc-6.0/doxygen/CC/html/index.html) +# **Further Information** +---------------------------- Other Links: * [EVIO Event Viewer on GitHub](https://github.com/JeffersonLab/JEventViewer) @@ -89,9 +93,7 @@ More information on the HIPO data format can be found at https://github.com/gava or from the CLAS12 Software Project Coordinator. ---------------------------- - # **Copyright** - ---------------------------- For any issues regarding use and copyright, read the [license](LICENSE.txt) file. From 21da8a57aa33256f5b623d29a0ba2847fc951fd8 Mon Sep 17 00:00:00 2001 From: Jon Zarling Date: Thu, 31 Jul 2025 12:13:51 -0500 Subject: [PATCH 05/14] More readme cleanup --- README.md | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 2c1e5e8eb..268725e7f 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,8 @@ # **EVIO 6 SOFTWARE PACKAGE** -EVIO stands for EVent Input/Output, a unique data format developed by Jefferson Lab. -It was created by the Data Acquisition (DAQ) group and is maintained by the -Experimental Physics Software and Computing Infrastructure (EPSCI) group at Thomas -Jefferson National Accelerator Facility (JLab). +EVIO stands for EVent Input/Output, a unique data format developed by Jefferson Lab used by typical detector readout systems at the lab. This software repository allows one to read & write `.evio` and `.ev` format data, within either a C/C++ or Java programming environment. -This software repository allows one to read & write `.evio` and `.ev` format data, -within either a C/C++ or Java programming environment. - -# **Getting Started** - ------------------------------ -## **Useful Links** ----------------------------- +# **Useful Links** Documentation on GitHub: @@ -26,6 +16,7 @@ Software Library Documentation: * [Doxygen for C Library](https://jeffersonlab.github.io/evio/doc-6.0/doxygen/C/html/index.html) * [Doxygen for C++ Libary](https://jeffersonlab.github.io/evio/doc-6.0/doxygen/CC/html/index.html) +# **Getting Started** ## **C/C++ Library** @@ -77,9 +68,11 @@ Requires Maven (`mvn`) and an installation of Java on your system. work. See downloads from [OpenJDK](https://openjdk.org/install/) or [Oracle](https://www.oracle.com/java/technologies/javase/jdk17-archive-downloads.html). ----------------------------- # **Further Information** ----------------------------- + + +The EVIO package was created by the Data Acquisition (DAQ) group and is maintained by the Experimental Physics Software and Computing Infrastructure (EPSCI) group at the Thomas Jefferson National Accelerator Facility (JLab). It has been developed by many authors over the years. + Other Links: * [EVIO Event Viewer on GitHub](https://github.com/JeffersonLab/JEventViewer) @@ -92,8 +85,8 @@ well as the software used to read and write to these respective `.evio` and `.hi More information on the HIPO data format can be found at https://github.com/gavalian/hipo, or from the CLAS12 Software Project Coordinator. ----------------------------- +Contact: Jon Zarling (jzarling@jlab.org) + # **Copyright** ----------------------------- For any issues regarding use and copyright, read the [license](LICENSE.txt) file. From a8d75ec530705345ab11b89c6ee923de5f5b1125 Mon Sep 17 00:00:00 2001 From: Jon Zarling Date: Fri, 8 Aug 2025 12:17:47 -0500 Subject: [PATCH 06/14] Java: ensure header recovery check has enough to do lookback. Separately, add close on error (to fix ide warnings). --- src/main/java/org/jlab/coda/jevio/EvioReader.java | 1 + .../java/org/jlab/coda/jevio/EvioReaderUnsyncV4.java | 2 +- src/utils/java/convertFormats.java | 11 +++++------ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/jlab/coda/jevio/EvioReader.java b/src/main/java/org/jlab/coda/jevio/EvioReader.java index 4bffadde7..60eedb597 100644 --- a/src/main/java/org/jlab/coda/jevio/EvioReader.java +++ b/src/main/java/org/jlab/coda/jevio/EvioReader.java @@ -215,6 +215,7 @@ public EvioReader(File file, boolean checkRecNumSeq, boolean sequential, boolean // Parse file header to find the file's endianness & evio version # if (findEvioVersion() != ReadStatus.SUCCESS) { + rFile.close(); throw new EvioException("Failed reading first block header"); } diff --git a/src/main/java/org/jlab/coda/jevio/EvioReaderUnsyncV4.java b/src/main/java/org/jlab/coda/jevio/EvioReaderUnsyncV4.java index 580f9c920..cc2970003 100644 --- a/src/main/java/org/jlab/coda/jevio/EvioReaderUnsyncV4.java +++ b/src/main/java/org/jlab/coda/jevio/EvioReaderUnsyncV4.java @@ -1046,7 +1046,7 @@ else if (bytesInBuf % 32768 == 0) { // Check block size, attempt to recover if flag set // (otherwise return exception with a hint to set flag) // System.out.println("blkSize BEFORE = " + blkSize); - if(doHeaderRecoveryCheck && fileSize - fileChannel.position() >= 10*4) { + if(doHeaderRecoveryCheck && (fileSize - fileChannel.position()) >= 10*4 && fileChannel.position() > 20*4) { int expectedMagicPos = 27; // in words int words_to_skip = 0; // words_to_skip = foundMagicPos - expectedMagicPos diff --git a/src/utils/java/convertFormats.java b/src/utils/java/convertFormats.java index 8bc86aa14..fb2d61203 100644 --- a/src/utils/java/convertFormats.java +++ b/src/utils/java/convertFormats.java @@ -12,13 +12,14 @@ public static void main(String[] args) throws IOException, EvioException { printUsage(); return; } - String outputFile = "merged.evio"; // default output name + String outputFile = "merged.evio"; // default output name java.util.List inputFiles = new java.util.ArrayList<>(); for (String arg : args) { if (arg.startsWith("-h")) { printUsage(); return; - } else if (arg.startsWith("-o")) { + } + else if (arg.startsWith("-o")) { // If option is "-oOutputName", set output file name outputFile = arg.substring(2); System.out.println("Output file: " + outputFile); @@ -27,10 +28,8 @@ public static void main(String[] args) throws IOException, EvioException { // System.out.println("Using current directory"); outputFile = "./" + outputFile; } - - } else { - inputFiles.add(arg); - } + } + else inputFiles.add(arg); } if (inputFiles.isEmpty()) { System.err.println("No input files specified."); From 0edfe783e29d2fcd6ccf08c9561fb4f857fc136e Mon Sep 17 00:00:00 2001 From: Jon Zarling Date: Fri, 8 Aug 2025 12:19:45 -0500 Subject: [PATCH 07/14] Cmake cleanup, add proper install target, plus update github actions --- .github/workflows/c_tests.yml | 2 +- CMakeLists.txt | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/c_tests.yml b/.github/workflows/c_tests.yml index 0aa39e785..e0d83ab76 100644 --- a/.github/workflows/c_tests.yml +++ b/.github/workflows/c_tests.yml @@ -17,7 +17,7 @@ jobs: run: cmake -S . -B build -DMAKE_EXAMPLES=1 - name: Build - run: cmake --build build -- -j$(nproc) + run: cmake --build build --target install --parallel - name: Run tests working-directory: build diff --git a/CMakeLists.txt b/CMakeLists.txt index f60244d27..ea8d53f14 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,22 +1,26 @@ +# See instructions in README.md for building and installing + +### VERSION DEFINITIONS ### cmake_minimum_required(VERSION 3.22) project(evio VERSION 6.1.0 LANGUAGES C CXX) - -# C/C++ standard and build options set(CMAKE_C_STANDARD 11) -set(CMAKE_C_STANDARD_REQUIRED ON) set(CMAKE_CXX_STANDARD 20) +# C/C++ build options +set(CMAKE_C_STANDARD_REQUIRED ON) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_DEBUG_POSTFIX -dbg) -add_compile_options(-Wall) +add_compile_options(-Wall) # Enable all warnings -# Build options (and defaults) +# Includes +include(GNUInstallDirs) +include(FindPackageHandleStandardArgs) # find_package_handle_standard_args() +include(CTest) + +# Build option parameters (and defaults) option(C_ONLY "SKIP building C++ library, build C only" OFF) option(MAKE_EXAMPLES "Build example/test programs" OFF) option(USE_FILESYSTEMLIB "Use C++ instead of Boost" OFF) option(DISRUPTOR_FETCH "Allow CMake to download Disruptor if not found" ON) -include(GNUInstallDirs) -include(FindPackageHandleStandardArgs) # find_package_handle_standard_args() -include(CTest) # Add custom find_package for Disruptor list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules") @@ -52,7 +56,9 @@ else() # Use default CMAKE_INSTALL_PREFIX set(INSTALL_DIR_DEFINED 1) set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_SOURCE_DIR}/${ARCH}) + set(CMAKE_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX}/include) message(STATUS "Installing to default location: ${CMAKE_INSTALL_PREFIX}") + message(STATUS "Installing includes to default location: ${CMAKE_INSTALL_INCLUDEDIR}") endif() # Boost libs From 5d05036e0de3efe14571c1dfc0206b0e7b0c3784 Mon Sep 17 00:00:00 2001 From: Jon Zarling Date: Fri, 8 Aug 2025 12:20:21 -0500 Subject: [PATCH 08/14] Update main README, add new ones for explaining folder contents --- README.md | 12 ++++++------ java/jars/README.md | 4 ++-- src/test/README.md | 6 ++++++ 3 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 src/test/README.md diff --git a/README.md b/README.md index 268725e7f..325837497 100644 --- a/README.md +++ b/README.md @@ -20,15 +20,14 @@ Software Library Documentation: ## **C/C++ Library** -To build C/C++ code from this repository: +The C and C++ libraries are build using `cmake`. To build C/C++ code from this repository: git clone https://github.com/JeffersonLab/evio/ cd evio; mkdir build cmake -S . -B build cmake --build build --target install --parallel -Note that during the cmake configure step (first of two `cmake` commands above), one can -toggle the following special flags: +Note that during the cmake configure step (first of two `cmake` commands above), one can also include the following special flags: * `C_ONLY` : build C lib only, skip C++ (default `-DC_ONLY=0`) * `MAKE_EXAMPLES`: build example/test programs (default `-DMAKE_EXAMPLES=0`) @@ -42,9 +41,10 @@ will not be supported in future releases. ### Prerequisites -C++ 17 or higher, `cmake`, `lz4`, `boost_system`, `boost_thread`, and `boost_chrono`. If LZ4 is not -already configured, it can be installed from [LZ4 on github](https://github.com/lz4/lz4). The boost -libraries are typically system-specific. +C++ 17 or higher, `cmake`, `lz4`, `boost_system`, `boost_thread`, and `boost_chrono`. Compilation can +be done using `clang` or `gcc` (gcc 11 or higher recommended). If LZ4 is not +already configured, it can be installed from [LZ4 on github](https://github.com/lz4/lz4). Installation of boost +libraries are typically system-specific (e.g. using a command like `yum`, `dbn`, `rpm`, `apt-get`, etc.). ## **Java Library** diff --git a/java/jars/README.md b/java/jars/README.md index b45124075..ba6120392 100644 --- a/java/jars/README.md +++ b/java/jars/README.md @@ -1,5 +1,5 @@ # **Java JARs Folder** -This folder contains any jar file dependencies required by the EVIO Java library that are not retrieved directly from the Maven repository. At present, only the disruptor library needs to be added from here. +This folder contains jar files used for executing EVIO code using the Java API. The main jar file was made using Java 17. -Note that the disruptor 4.0.0 jar file included comes from a [JeffersonLab fork](https://github.com/JeffersonLab/disruptor) of the original lmax Java library. This forked repository adds an additional function (SpinCountBackoffWaitStrategy) that is required in the evio Java Library. \ No newline at end of file +It also includes dependencies required by the EVIO Java library that are not retrieved directly from the Maven repository. At present, only the disruptor dependency needs to be added from here. Note that the disruptor 4.0.0 jar file included comes from a [JeffersonLab fork](https://github.com/JeffersonLab/disruptor) of the original lmax Java library. This forked repository adds an additional function (SpinCountBackoffWaitStrategy) that is required in the evio Java Library. \ No newline at end of file diff --git a/src/test/README.md b/src/test/README.md new file mode 100644 index 000000000..edc9de5a8 --- /dev/null +++ b/src/test/README.md @@ -0,0 +1,6 @@ +# **Test Programs** + +These programs are used for continuous integration (CI) tests. As such, +they may not be the most useful reference programs for users. Essentially, +they test the various C++ and Java APIs to ensure that we can write some +simple dummy events to an evio file and read back expected values. From d49b56490413ec8462a0b2a06706ebef64de2e56 Mon Sep 17 00:00:00 2001 From: Jon Zarling Date: Fri, 8 Aug 2025 12:31:46 -0500 Subject: [PATCH 09/14] Don't quit if one encounters empty BANK/SEGMENT/TAGSEGMENT, just ignore instead. Now Hall D files should read. --- src/libsrc++/EventParser.cpp | 6 ++++-- src/utils/cpp/evio_merge_files.cpp | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libsrc++/EventParser.cpp b/src/libsrc++/EventParser.cpp index bfd50de6f..4d937b1fb 100644 --- a/src/libsrc++/EventParser.cpp +++ b/src/libsrc++/EventParser.cpp @@ -227,9 +227,11 @@ namespace evio { // --which will be interpreted by the various "get data" methods. auto & bytes = structure->getRawBytes(); ByteOrder byteOrder = structure->getByteOrder(); - if (bytes.empty()) { - throw EvioException("Null data in structure"); + // throw EvioException("Null data in structure"); + // printf("WARNING: no data inside structure! Skipping... \n"); + // printf(" structure %s \n", structure->getHeader()->toString().c_str()); + return; } size_t length = bytes.size(); diff --git a/src/utils/cpp/evio_merge_files.cpp b/src/utils/cpp/evio_merge_files.cpp index fb8d68fa7..cec82a71b 100644 --- a/src/utils/cpp/evio_merge_files.cpp +++ b/src/utils/cpp/evio_merge_files.cpp @@ -150,6 +150,8 @@ void Process(unsigned int &NEvents, unsigned int &NEvents_read) std::shared_ptr event; while ((event = reader.parseNextEvent())) { + // for (uint32_t i = 4; i < reader.getEventCount(); i++) { // Non-sequential way to read + // std::shared_ptr event = reader.parseEvent(i); writer->writeEvent(event); NEvents_read++; NEvents++; From 53480ce24fa5e7df84edf0ea14ea445d5fe34249 Mon Sep 17 00:00:00 2001 From: Jon Zarling Date: Fri, 8 Aug 2025 12:35:25 -0500 Subject: [PATCH 10/14] C++ program that prints evio contents (initial version) --- src/utils/cpp/evio_print_contents_draft.cpp | 92 +++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 src/utils/cpp/evio_print_contents_draft.cpp diff --git a/src/utils/cpp/evio_print_contents_draft.cpp b/src/utils/cpp/evio_print_contents_draft.cpp new file mode 100644 index 000000000..7ab3d4919 --- /dev/null +++ b/src/utils/cpp/evio_print_contents_draft.cpp @@ -0,0 +1,92 @@ +#include +#include + +#include "eviocc.h" // EVIO C++ API all inside + +// Use the EVIO namespace for convenience +using namespace evio; + +// TODO: add max_events as command line argument, test for V6 as well +// TODO: and also print more than uint32s + +int main(int argc, char* argv[]) { + if (argc < 2) { + std::cerr << "Usage: " << argv[0] << " " << std::endl; + return 1; + } + std::string filename = argv[1]; + + int maxEvents = 10; // Maximum number of events to read + + try { + // Open the EVIO6 file using EvioReader + EvioReader reader(filename); + + // Check for an embedded XML dictionary + if (reader.hasDictionaryXML()) { + // Retrieve the raw XML dictionary text + std::string xmlDict = reader.getDictionaryXML(); + // Print the XML dictionary to standard output + std::cout << xmlDict; + if (!xmlDict.empty() && xmlDict.back() != '\n') std::cout << std::endl; // ensure a newline at end, if not already present + } else { + // No dictionary present in the file + std::cout << "No XML dictionary found." << std::endl; + } + + std::cout << "EVIO Ver: " << reader.getEvioVersion() << std::endl; + std::cout << "File Size: " << reader.fileSize() << std::endl; + std::cout << "Has first event?: " << reader.hasFirstEvent() << std::endl; + std::cout << "Getting event count... this may take a while for large files " << std::endl; + std::cout << "Event count: " << reader.getEventCount() << std::endl; + + for (uint32_t i = 1; i < reader.getEventCount(); i++) { + if(i+1 >= maxEvents) { + std::cout << "Reached maximum number of events to read: " << maxEvents << std::endl; + break; + } + std::shared_ptr ev = reader.parseEvent(i+1); + std::cout << " got & parsed ev " << (i+1) << std::endl; + std::cout << " event ->\n" << ev->toString() << std::endl; + auto& dataVec = ev->getRawBytes(); + std::cout << "Event has tag = " << ev->getHeader()->getTag() << std::endl; + std::cout << "Event structure type = " << ev->getStructureType().toString() << std::endl; + std::vector< std::shared_ptr< BaseStructure > > & children = ev->getChildren(); + std::cout << "Event has " << children.size() << " children" << std::endl; + std::shared_ptr< BaseStructure > child1; + if(children.size()>0) child1 = children[0]; + for (size_t j = 0; j < children.size(); j++) { + std::cout << "Child " << j << " tag = " << children[j]->getStructureType().toString() << std::endl; + std::cout << "NChildren: " << children[j]->getChildCount() << std::endl; + + for (size_t k = 0; k < children[j]->getChildCount(); k++) { + std::cout << "Child " << j << ", subchild " << k << " " << children[j]->getChildAt(k)->getStructureType().toString() << std::endl; + std::cout << "Subchild " << k << " datatype: " << children[j]->getChildAt(k)->getHeader()->getDataType().toString() << std::endl; + std::cout << "nsubchildren: " << children[j]->getChildAt(k)->getChildCount() << std::endl; + std::cout << "num items stored: " << children[j]->getChildAt(k)->getNumberDataItems() << std::endl; + if(children[j]->getChildAt(k)->getHeader()->getDataType() == DataType::UINT32) { + std::cout << "Data: "; + std::vector< uint32_t > data_uint_vec = children[j]->getChildAt(k)->getUIntData(); + for (size_t l = 0; l < data_uint_vec.size(); l++) { + std::cout << data_uint_vec[l] << " "; + } + std::cout << std::endl; + } + } + std::cout << std::endl; + } + std::cout << "End of event" << std::endl << std::endl; + + // Util::printBytes(dataVec.data(), dataVec.size()," Event #" + std::to_string(i+1)); + } + + + + + } catch (const std::exception& e) { + std::cerr << "Error: Unable to read EVIO file. " << e.what() << std::endl; + return 1; + } + + return 0; +} From d4413b6bacb69e94d14f2bbec118655422d2d55d Mon Sep 17 00:00:00 2001 From: Jon Zarling Date: Fri, 8 Aug 2025 13:01:41 -0500 Subject: [PATCH 11/14] Rename and fix output filename --- src/utils/java/{convertFormats.java => evio_merge_files.java} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename src/utils/java/{convertFormats.java => evio_merge_files.java} (97%) diff --git a/src/utils/java/convertFormats.java b/src/utils/java/evio_merge_files.java similarity index 97% rename from src/utils/java/convertFormats.java rename to src/utils/java/evio_merge_files.java index fb2d61203..a63cad719 100644 --- a/src/utils/java/convertFormats.java +++ b/src/utils/java/evio_merge_files.java @@ -4,7 +4,7 @@ import java.io.IOException; import java.nio.ByteOrder; -public class convertFormats { +public class evio_merge_files { public static void main(String[] args) throws IOException, EvioException { // Parse command-line arguments if (args.length < 1) { @@ -12,7 +12,7 @@ public static void main(String[] args) throws IOException, EvioException { printUsage(); return; } - String outputFile = "merged.evio"; // default output name + String outputFile = "./merged.evio"; // default output name java.util.List inputFiles = new java.util.ArrayList<>(); for (String arg : args) { if (arg.startsWith("-h")) { From eb460fd45566c8de2bce891356734e7c1882b517 Mon Sep 17 00:00:00 2001 From: Jon Zarling Date: Mon, 11 Aug 2025 13:24:17 -0500 Subject: [PATCH 12/14] Fix merge program to support V6 files (without compression in output) --- src/utils/cpp/evio_merge_files.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/utils/cpp/evio_merge_files.cpp b/src/utils/cpp/evio_merge_files.cpp index cec82a71b..e0df523fd 100644 --- a/src/utils/cpp/evio_merge_files.cpp +++ b/src/utils/cpp/evio_merge_files.cpp @@ -118,7 +118,7 @@ void Process(unsigned int &NEvents, unsigned int &NEvents_read) std::string dictXml = ""; try { if (!INFILENAMES.empty()) { - EvioReaderV4 dictReader(INFILENAMES[0]); + EvioReader dictReader(INFILENAMES[0]); if (dictReader.hasDictionaryXML()) { dictXml = dictReader.getDictionaryXML(); std::cout << "Dictionary found in first input file.\n"; @@ -131,7 +131,7 @@ void Process(unsigned int &NEvents, unsigned int &NEvents_read) } // Set up EVIO4 writer - std::unique_ptr writer = std::make_unique( + std::unique_ptr writer = std::make_unique( outFile, "", "", 1, 0, maxRecordBytes, maxEventsPerRecord, @@ -139,6 +139,7 @@ void Process(unsigned int &NEvents, unsigned int &NEvents_read) dictXml, true, false, nullptr, 1, 0, 1, 1, + Compressor::UNCOMPRESSED, 1, 0, bufferBytes ); @@ -146,11 +147,11 @@ void Process(unsigned int &NEvents, unsigned int &NEvents_read) for (auto filename : INFILENAMES) { try { std::cout << "Opening input file: " << filename << std::endl; - EvioReaderV4 reader(filename); + EvioReader reader(filename); std::shared_ptr event; - while ((event = reader.parseNextEvent())) { - // for (uint32_t i = 4; i < reader.getEventCount(); i++) { // Non-sequential way to read + while ((event = reader.parseNextEvent())) { // Sequential read method + // for (uint32_t i = 4; i < reader.getEventCount(); i++) { // Non-sequential method // std::shared_ptr event = reader.parseEvent(i); writer->writeEvent(event); NEvents_read++; From baa719f73dd9c90d166c597860cb23dc77a40403 Mon Sep 17 00:00:00 2001 From: Jon Zarling Date: Mon, 11 Aug 2025 13:25:57 -0500 Subject: [PATCH 13/14] BIG memory leak fix. Put bytes into vector instead of array. --- src/libsrc++/EvioReaderV4.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libsrc++/EvioReaderV4.cpp b/src/libsrc++/EvioReaderV4.cpp index 9a2d207fe..18d66bd7c 100644 --- a/src/libsrc++/EvioReaderV4.cpp +++ b/src/libsrc++/EvioReaderV4.cpp @@ -1090,7 +1090,7 @@ namespace evio { uint32_t eventDataSizeBytes = 4*(length - 1); try { - auto *bytes = new uint8_t[eventDataSizeBytes]; + auto bytes = std::vector(eventDataSizeBytes); uint32_t bytesToGo = eventDataSizeBytes; uint32_t offset = 0; @@ -1106,7 +1106,7 @@ namespace evio { blkBytesRemaining : bytesToGo; // Read in bytes remaining in internal buffer - byteBuffer->getBytes(bytes + offset, bytesToReadNow); + byteBuffer->getBytes(bytes.data() + offset, bytesToReadNow); offset += bytesToReadNow; bytesToGo -= bytesToReadNow; blkBytesRemaining -= bytesToReadNow; @@ -1123,12 +1123,14 @@ namespace evio { blkBytesRemaining = blockBytesRemaining(); } } + + } // Last (perhaps only) read - byteBuffer->getBytes(bytes + offset, bytesToGo); + byteBuffer->getBytes(bytes.data() + offset, bytesToGo); //std::cout << "nextEvent: eventDataSizeByte = " << eventDataSizeBytes << std::endl; - event->setRawBytes(bytes, eventDataSizeBytes); + event->setRawBytes(bytes.data(), eventDataSizeBytes); event->setByteOrder(byteOrder); // add this to track endianness, timmer // Don't worry about dictionaries here as version must be 1-3 event->setEventNumber(++eventNumber); From 04fe2ea77eddcfb5698cf56689162d916f9a1d6c Mon Sep 17 00:00:00 2001 From: jonzarling Date: Wed, 10 Sep 2025 17:14:57 -0500 Subject: [PATCH 14/14] Remove unfinished utility program before merging into main (but only on this tmp branch) --- src/utils/cpp/evio_print_contents_draft.cpp | 92 --------------------- 1 file changed, 92 deletions(-) delete mode 100644 src/utils/cpp/evio_print_contents_draft.cpp diff --git a/src/utils/cpp/evio_print_contents_draft.cpp b/src/utils/cpp/evio_print_contents_draft.cpp deleted file mode 100644 index 7ab3d4919..000000000 --- a/src/utils/cpp/evio_print_contents_draft.cpp +++ /dev/null @@ -1,92 +0,0 @@ -#include -#include - -#include "eviocc.h" // EVIO C++ API all inside - -// Use the EVIO namespace for convenience -using namespace evio; - -// TODO: add max_events as command line argument, test for V6 as well -// TODO: and also print more than uint32s - -int main(int argc, char* argv[]) { - if (argc < 2) { - std::cerr << "Usage: " << argv[0] << " " << std::endl; - return 1; - } - std::string filename = argv[1]; - - int maxEvents = 10; // Maximum number of events to read - - try { - // Open the EVIO6 file using EvioReader - EvioReader reader(filename); - - // Check for an embedded XML dictionary - if (reader.hasDictionaryXML()) { - // Retrieve the raw XML dictionary text - std::string xmlDict = reader.getDictionaryXML(); - // Print the XML dictionary to standard output - std::cout << xmlDict; - if (!xmlDict.empty() && xmlDict.back() != '\n') std::cout << std::endl; // ensure a newline at end, if not already present - } else { - // No dictionary present in the file - std::cout << "No XML dictionary found." << std::endl; - } - - std::cout << "EVIO Ver: " << reader.getEvioVersion() << std::endl; - std::cout << "File Size: " << reader.fileSize() << std::endl; - std::cout << "Has first event?: " << reader.hasFirstEvent() << std::endl; - std::cout << "Getting event count... this may take a while for large files " << std::endl; - std::cout << "Event count: " << reader.getEventCount() << std::endl; - - for (uint32_t i = 1; i < reader.getEventCount(); i++) { - if(i+1 >= maxEvents) { - std::cout << "Reached maximum number of events to read: " << maxEvents << std::endl; - break; - } - std::shared_ptr ev = reader.parseEvent(i+1); - std::cout << " got & parsed ev " << (i+1) << std::endl; - std::cout << " event ->\n" << ev->toString() << std::endl; - auto& dataVec = ev->getRawBytes(); - std::cout << "Event has tag = " << ev->getHeader()->getTag() << std::endl; - std::cout << "Event structure type = " << ev->getStructureType().toString() << std::endl; - std::vector< std::shared_ptr< BaseStructure > > & children = ev->getChildren(); - std::cout << "Event has " << children.size() << " children" << std::endl; - std::shared_ptr< BaseStructure > child1; - if(children.size()>0) child1 = children[0]; - for (size_t j = 0; j < children.size(); j++) { - std::cout << "Child " << j << " tag = " << children[j]->getStructureType().toString() << std::endl; - std::cout << "NChildren: " << children[j]->getChildCount() << std::endl; - - for (size_t k = 0; k < children[j]->getChildCount(); k++) { - std::cout << "Child " << j << ", subchild " << k << " " << children[j]->getChildAt(k)->getStructureType().toString() << std::endl; - std::cout << "Subchild " << k << " datatype: " << children[j]->getChildAt(k)->getHeader()->getDataType().toString() << std::endl; - std::cout << "nsubchildren: " << children[j]->getChildAt(k)->getChildCount() << std::endl; - std::cout << "num items stored: " << children[j]->getChildAt(k)->getNumberDataItems() << std::endl; - if(children[j]->getChildAt(k)->getHeader()->getDataType() == DataType::UINT32) { - std::cout << "Data: "; - std::vector< uint32_t > data_uint_vec = children[j]->getChildAt(k)->getUIntData(); - for (size_t l = 0; l < data_uint_vec.size(); l++) { - std::cout << data_uint_vec[l] << " "; - } - std::cout << std::endl; - } - } - std::cout << std::endl; - } - std::cout << "End of event" << std::endl << std::endl; - - // Util::printBytes(dataVec.data(), dataVec.size()," Event #" + std::to_string(i+1)); - } - - - - - } catch (const std::exception& e) { - std::cerr << "Error: Unable to read EVIO file. " << e.what() << std::endl; - return 1; - } - - return 0; -}