diff --git a/SEFramework/SEFramework/FITS/FitsImageSource.h b/SEFramework/SEFramework/FITS/FitsImageSource.h index 5686df98c..a66107d63 100644 --- a/SEFramework/SEFramework/FITS/FitsImageSource.h +++ b/SEFramework/SEFramework/FITS/FitsImageSource.h @@ -46,6 +46,7 @@ using Euclid::FilePool::FileHandler; class FitsImageSource : public ImageSource, public std::enable_shared_from_this { public: + static std::shared_ptr getFileManager(unsigned int max_open_files = 500); /** * Constructor @@ -57,14 +58,14 @@ class FitsImageSource : public ImageSource, public std::enable_shared_from_this< */ explicit FitsImageSource(const std::string& filename, int hdu_number = 0, ImageTile::ImageType image_type = ImageTile::AutoType, - std::shared_ptr manager = FileManager::getDefault()); + std::shared_ptr manager = getFileManager()); FitsImageSource(const std::string& filename, int width, int height, ImageTile::ImageType image_type, const std::shared_ptr coord_system = nullptr, bool append = false, bool empty_primary = false, - std::shared_ptr manager = FileManager::getDefault()); + std::shared_ptr manager = getFileManager()); virtual ~FitsImageSource() = default; @@ -137,6 +138,8 @@ class FitsImageSource : public ImageSource, public std::enable_shared_from_this< ImageTile::ImageType m_image_type; int m_current_layer; + + static std::shared_ptr s_file_manager; }; } diff --git a/SEFramework/src/lib/FITS/FitsImageSource.cpp b/SEFramework/src/lib/FITS/FitsImageSource.cpp index f68b06297..776ac420b 100644 --- a/SEFramework/src/lib/FITS/FitsImageSource.cpp +++ b/SEFramework/src/lib/FITS/FitsImageSource.cpp @@ -23,10 +23,13 @@ */ #include "SEFramework/FITS/FitsImageSource.h" + #include "SEFramework/FITS/FitsFile.h" #include "SEUtils/VariantCast.h" #include +#include #include +#include "ElementsKernel/Logging.h" #include #include #include @@ -39,6 +42,9 @@ namespace SourceXtractor { +static Elements::Logging logger = Elements::Logging::getLogger("FitsFile"); + + using Euclid::make_unique; namespace { @@ -386,7 +392,18 @@ int FitsImageSource::getImageType() const { return LONGLONG_IMG; } } - //FIXME add missing types + +std::shared_ptr FitsImageSource::getFileManager(unsigned int max_open_files) { + if (!s_file_manager) { + logger.info() << "Creating shared LRUFileManager with limit of " << max_open_files << " open files."; + s_file_manager = std::make_shared(max_open_files); + } + return s_file_manager; } + +std::shared_ptr FitsImageSource::s_file_manager = nullptr; + +} // namespace SourceXtractor + diff --git a/SEImplementation/SEImplementation/Configuration/FileManagerConfig.h b/SEImplementation/SEImplementation/Configuration/FileManagerConfig.h new file mode 100644 index 000000000..08f1e7d80 --- /dev/null +++ b/SEImplementation/SEImplementation/Configuration/FileManagerConfig.h @@ -0,0 +1,44 @@ +/** Copyright © 2019-2025 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SEIMPLEMENTATION_CONFIGURATION_FILEMANAGERCONFIG_H +#define SEIMPLEMENTATION_CONFIGURATION_FILEMANAGERCONFIG_H + +#include "Configuration/Configuration.h" + +namespace SourceXtractor { + +class FileManagerConfig : public Euclid::Configuration::Configuration { +public: + explicit FileManagerConfig(long manager_id); + virtual ~FileManagerConfig() = default; + + std::map getProgramOptions() override; + void preInitialize(const UserValues& args) override; + void initialize(const UserValues& args) override; + + int getMaxSimultaneousFiles() const { + return m_max_simultaneous_files; + } + +private: + int m_max_simultaneous_files; +}; + +} // namespace SourceXtractor + +#endif // SEIMPLEMENTATION_CONFIGURATION_FILEMANAGERCONFIG_H \ No newline at end of file diff --git a/SEImplementation/src/lib/Configuration/FileManagerConfig.cpp b/SEImplementation/src/lib/Configuration/FileManagerConfig.cpp new file mode 100644 index 000000000..e37c683c9 --- /dev/null +++ b/SEImplementation/src/lib/Configuration/FileManagerConfig.cpp @@ -0,0 +1,58 @@ +/** Copyright © 2019-2025 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEImplementation/Configuration/FileManagerConfig.h" + +#include + +#include "ElementsKernel/Exception.h" +#include "Configuration/ConfigManager.h" + +#include "SEFramework/FITS/FitsImageSource.h" + +#include + +using namespace Euclid::Configuration; +namespace po = boost::program_options; + +namespace SourceXtractor { + +static const std::string MAX_SIMULTANEOUS_FILES {"max-simultaneous-files" }; + + + // Constructor +FileManagerConfig::FileManagerConfig(long manager_id) + : Configuration(manager_id), m_max_simultaneous_files(500) { +} + +std::map FileManagerConfig::getProgramOptions() { + return { {"File Manager", { + {MAX_SIMULTANEOUS_FILES.c_str(), po::value()->default_value(500), + "Maximum number of simultaneous open files."} + }} }; +} + +void FileManagerConfig::preInitialize(const UserValues& args) { + // We have to configure the FileManager before any other configuration tries to use it + m_max_simultaneous_files = args.find(MAX_SIMULTANEOUS_FILES)->second.as(); + FitsImageSource::getFileManager(m_max_simultaneous_files); +} + +void FileManagerConfig::initialize(const UserValues& args) { +} + +} // namespace SourceXtractor \ No newline at end of file diff --git a/SEMain/src/program/SourceXtractor.cpp b/SEMain/src/program/SourceXtractor.cpp index 49b2d4937..e79ae077e 100644 --- a/SEMain/src/program/SourceXtractor.cpp +++ b/SEMain/src/program/SourceXtractor.cpp @@ -74,6 +74,7 @@ #include "SEImplementation/Configuration/MemoryConfig.h" #include "SEImplementation/Configuration/OutputConfig.h" #include "SEImplementation/Configuration/SamplingConfig.h" +#include "SEImplementation/Configuration/FileManagerConfig.h" #include "SEImplementation/CheckImages/CheckImages.h" #include "SEImplementation/Prefetcher/Prefetcher.h" @@ -191,6 +192,7 @@ class SEMain : public Elements::Program { config_manager.registerConfiguration(); config_manager.registerConfiguration(); config_manager.registerConfiguration(); + config_manager.registerConfiguration(); CheckImages::getInstance().reportConfigDependencies(config_manager); @@ -319,15 +321,15 @@ class SEMain : public Elements::Program { throw Elements::Exception() << "The configuration file '" << cfg_file << "' does not exist"; } } - + // Create the progress listener and printer ASAP progress_printer_factory.configure(args); auto progress_mediator = progress_printer_factory.createProgressMediator(); - + // Initialize the rest of the components auto& config_manager = ConfigManager::getInstance(config_manager_id); config_manager.initialize(args); - + // Configure TileManager auto memory_config = config_manager.getConfiguration(); TileManager::getInstance()->setOptions(memory_config.getTileSize(),