Nymph2 2/bind dataframe

CMakeLists.txt

@@ -66,9 +66,9 @@ set( PRIVATE_EXT_LIBS )
 #######
 
 # Boost (1.46 required for filesystem version 3)
-list( APPEND BOOST_COMPONENTS chrono date_time filesystem program_options system thread )
+list( APPEND BOOST_COMPONENTS chrono date_time filesystem system )
 find_package( Boost 1.46.0 REQUIRED COMPONENTS ${BOOST_COMPONENTS} )
-list( APPEND PUBLIC_EXT_LIBS Boost::boost Boost::chrono Boost::date_time Boost::program_options Boost::thread Boost::filesystem )
+list( APPEND PUBLIC_EXT_LIBS Boost::boost Boost::chrono Boost::date_time Boost::filesystem )
 # make sure dynamic linking is assumed for all boost libraries
 set( Boost_USE_STATIC_LIBS OFF )
 set( Boost_USE_STATIC_RUNTIME OFF )

Cpp/Library/Data/Data.cc

@@ -7,6 +7,8 @@
 
 #include "Data.hh"
 
+#include "typename.hh"
+
 namespace Nymph
 {
     Data::Data()
@@ -15,4 +17,9 @@ namespace Nymph
     Data::~Data()
     {}
 
+    std::string Data::GetName() const
+    {
+        return scarab::type(*this);
+    }
+
 } /* namespace Nymph */

Cpp/Library/Data/Data.hh

@@ -9,6 +9,8 @@
 #ifndef NYMPH_DATA_HH_
 #define NYMPH_DATA_HH_
 
+#include <string>
+
 namespace Nymph
 {
 
@@ -17,6 +19,8 @@ namespace Nymph
         public:
             Data();
             virtual ~Data();
+
+            std::string GetName() const;
     };
 
 } /* namespace Nymph */

Cpp/Library/Data/DataFrame.cc

@@ -9,11 +9,95 @@
 
 namespace Nymph
 {
+    IndexableData::IndexableData( std::unique_ptr< Data >&& dataPtr, std::type_index&& typeIndex, std::string&& typeName ) :
+            fDataPtr( std::move(dataPtr) ),
+            fTypeIndex( std::move(typeIndex) ),
+            fTypeName( std::move(typeName) )
+    {}
+
+
     DataFrame::DataFrame() :
-        fDataObjects()
+        fDataObjects(),
+        fDataMapByType( fDataObjects.get<type_t>() ),
+        fDataMapByTypeName( fDataObjects.get<tname_t>() )
     {}
 
     DataFrame::~DataFrame()
     {}
 
+    bool DataFrame::Has( const std::string& typeName ) const
+    {
+        return fDataMapByTypeName.count( typeName ) != 0;
+    }
+
+    Data& DataFrame::Get( const std::string& typeName )
+    {
+        auto iter = fDataMapByTypeName.find( typeName );
+        if( iter == fDataMapByTypeName.end() )
+        {
+            THROW_EXCEPT_HERE( DataFrameException() << "Adding event type by Get(string) is not yet implemented" );
+/*            // TODO: this doesn't work; needs mechanism to go from string to type
+            auto result = fDataMapByTypeName.insert( IndexableData::Create<XDataNoConst>() );
+            if( result.second )
+            {
+                return static_cast< XDataNoConst& >( *result.first->fDataPtr );
+            }
+            else
+            {
+                THROW_EXCEPT_HERE( DataFrameException() << "Data type called <" << typeName << "> could not be added to the data frame" );
+            }
+*/        }
+        return *iter->fDataPtr;
+    }
+
+    const Data& DataFrame::Get( const std::string& typeName ) const
+    {
+        auto iter = fDataMapByTypeName.find( typeName );
+        if( iter == fDataMapByTypeName.end() )
+        {
+            THROW_EXCEPT_HERE( DataFrameException() << "Data type called <" << typeName << "> is not present when const Get() was called" );
+        }
+        return *iter->fDataPtr;
+    }
+
+    void DataFrame::Set( const std::string& typeName, Data* ptr )
+    {
+        // Note: takes ownership of ptr
+        DoSet( IndexableData( std::unique_ptr<Data>(ptr), typeid(*ptr), scarab::type(*ptr) ) );
+        return;
+    }
+
+    void DataFrame::Set( const std::string& typeName, std::unique_ptr<Data>&& ptr )
+    {
+        // Note: takes ownership of object pointed to by ptr
+        auto& ref = *ptr.get();
+        DoSet( IndexableData( std::move(ptr), typeid(ref), scarab::type(*ptr) ) );
+        return;
+    }
+
+    void DataFrame::DoSet( IndexableData&& indData )
+    {
+        auto iter = fDataMapByType.find( indData.fTypeIndex );
+        if( iter == fDataMapByType.end() )
+        {
+            auto result = fDataMapByType.insert( std::move(indData) );
+            if( ! result.second )
+            {
+                THROW_EXCEPT_HERE( DataFrameException() << "Attempt to insert new data object failed for data type named <" << indData.fTypeName << ">" );
+            }
+            return;
+        }
+        if( ! fDataMapByType.replace( iter, std::move(indData) ) )
+        {
+            THROW_EXCEPT_HERE( DataFrameException() << "Attempt to replace data object failed for data type <" << indData.fTypeName << ">" );
+        }
+        return;
+    }
+
+    void DataFrame::Remove( const std::string& typeName )
+    {
+        fDataMapByTypeName.erase( typeName );
+        return;
+    }
+
 } /* namespace Nymph */

Cpp/Library/Data/DataFrame.hh

@@ -16,10 +16,16 @@
 
 #include "typename.hh"
 
-#include <unordered_map>
+#include <boost/multi_index_container.hpp>
+#include <boost/multi_index/hashed_index.hpp>
+#include <boost/multi_index/member.hpp>
+#include <boost/multi_index/key.hpp>
+
 #include <memory>
 #include <typeindex>
 
+namespace bmi = ::boost::multi_index;
+
 namespace Nymph
 {
     class DataFrameException : public scarab::typed_exception< DataFrameException >
@@ -29,6 +35,37 @@ namespace Nymph
             ~DataFrameException() = default;
     };
 
+    // map< std::string, std::type_index > may give us what we need to go from string to class
+    //  * this would enable DataFrame::Set( std::string, Data* ) and DataFrame::Set( std::string, std::unique_ptr<Data> )
+    //  * would not enable DataFrame::Get(); need to be able to create an instance of the type
+
+    /// Struct to hold individual data pointers and indexing information for the DataFrame
+    struct IndexableData
+    {
+        std::unique_ptr< Data > fDataPtr;
+        std::type_index fTypeIndex;
+        std::string fTypeName;
+
+        IndexableData( std::unique_ptr< Data >&& dataPtr, std::type_index&& typeIndex, std::string&& typeName );
+
+        // Use factory functions so that we can template it with the derived data type 
+        // (wouldn't be able to have the empty constructor beacuse there's nothing to determine the template type by, 
+        //  and you can't have a template argument for a constructor)
+
+        /// Create an IndexableData with an empty object of type XData
+        template< typename XData >
+        static IndexableData Create();
+        /// Create an IndexableData with data (claims ownership of that object)
+        template< typename XData >
+        static IndexableData Create( XData* data );
+        /// Create an IndexableData with the object pointed to by dataPtr (claims ownership of that object)
+        template< typename XData >
+        static IndexableData Create( std::unique_ptr< XData >&& dataPtr );
+        /// Create an IndexableData with a copy of the provided object
+        template< typename XData >
+        static IndexableData Create( const XData& data );
+    };
+
     // forward declare so we can define DataHandle here
     class DataFrame;
 
@@ -58,6 +95,10 @@ namespace Nymph
             /// Returns true if the frame has no data objects
             bool Empty() const;
 
+            //**********************
+            // Type-based interface
+            //**********************
+
             /// Returns true if object of type(s) XData exist in the frame; returns false otherwise; Has<>() returns true
             template< typename... XData >
             bool Has() const;
@@ -92,16 +133,90 @@ namespace Nymph
             template< typename XData >
             void Remove();
 
+
+            //************************
+            // String-based interface
+            //************************
+
+            bool Has( const std::string& typeName ) const;
+
+            Data& Get( const std::string& typeName );
+
+            const Data& Get( const std::string& typeName ) const;
+
+            void Set( const std::string& typeName, Data* ptr );
+
+            void Set( const std::string& typeName, std::unique_ptr<Data>&& ptr );
+
+            //void Set( const std::string& typeName, const Data& obj );  <-- this probably requires having Clone() functions setup in Data
+
+            void Remove( const std::string& typeName );
+
+
+            //*********
+            // Storage
+            //*********
+
             // typedef used to avoid problems with the comma in the MEMVAR macro
-            typedef std::unordered_map< std::type_index, std::unique_ptr<Data> > DataMap;
+            //typedef std::unordered_map< std::type_index, std::unique_ptr<Data> > DataMap;
+            // tags
+            struct type_t{};
+            struct tname_t{};
+            typedef bmi::multi_index_container< 
+                IndexableData,
+                bmi::indexed_by<
+                    bmi::hashed_unique< bmi::tag<type_t>, bmi::key< &IndexableData::fTypeIndex > >,
+                    bmi::hashed_unique< bmi::tag<tname_t>, bmi::key< &IndexableData::fTypeName > >
+                >
+            > DataMap;
+            typedef DataMap::index<type_t>::type DataMapByType;
+            typedef DataMap::index<tname_t>::type DataMapByTypeName;
             MEMVAR_REF( DataMap, DataObjects );
 
         protected:
             template< typename XData >
             bool HasOneType() const;
+
+            void DoSet( IndexableData&& indData );
+
+            DataMapByType& fDataMapByType;
+            DataMapByTypeName& fDataMapByTypeName;
     };
 
 
+    //*************************
+    // IndexableData functions
+    //*************************
+
+    template< typename XData >
+    IndexableData IndexableData::Create()
+    {
+        return IndexableData{ std::make_unique<XData>(), typeid(XData), scarab::type<XData>() };
+    }
+
+    template< typename XData >
+    IndexableData IndexableData::Create( XData* data )
+    {
+        return IndexableData{ std::unique_ptr<XData>(data), typeid(XData), scarab::type<XData>() };
+    }
+
+    template< typename XData >
+    IndexableData IndexableData::Create( std::unique_ptr< XData >&& dataPtr )
+    {
+        return IndexableData{ std::move(dataPtr), typeid(XData), scarab::type<XData>() };
+    }
+
+    template< typename XData >
+    IndexableData IndexableData::Create( const XData& data )
+    {
+        return IndexableData{ std::make_unique<XData>(data), typeid(XData), scarab::type<XData>() };
+    }
+
+
+    //*********************
+    // DataFrame functions
+    //*********************
+
     inline bool DataFrame::Empty() const
     {
         return fDataObjects.empty();
@@ -117,61 +232,73 @@ namespace Nymph
     bool DataFrame::HasOneType() const
     {
         typedef std::remove_const_t< XData > XDataNoConst;
-        if( fDataObjects.count( typeid(XDataNoConst) ) == 0 ) return false;
-        return true;
+        return fDataMapByType.count( typeid(XDataNoConst) ) != 0;
     }
 
     template< typename XData >
     XData& DataFrame::Get()
     {
         typedef std::remove_const_t< XData > XDataNoConst;
-        if( ! Has< XDataNoConst >() )
+        auto iter = fDataMapByType.find( typeid(XDataNoConst) );
+        if( iter == fDataMapByType.end() )
         {
-            fDataObjects[ typeid(XDataNoConst) ].reset( new XDataNoConst() );
+            auto result = fDataMapByType.insert( IndexableData::Create<XDataNoConst>() );
+            if( result.second )
+            {
+                return static_cast< XDataNoConst& >( *result.first->fDataPtr );
+            }
+            else
+            {
+                THROW_EXCEPT_HERE( DataFrameException() << "Data type <" << scarab::type<XDataNoConst>() << "> could not be added to the data frame" );
+            }
         }
-        return static_cast< XDataNoConst& >( *fDataObjects[typeid(XDataNoConst)] );
+        return static_cast< XDataNoConst& >( *iter->fDataPtr );
     }
 
     template< typename XData >
     const XData& DataFrame::Get() const
     {
         typedef std::remove_const_t< XData > XDataNoConst;
-        if( Has< XDataNoConst >() )
+        auto iter = fDataMapByType.find( typeid(XDataNoConst) );
+        if( iter == fDataMapByType.end() )
         {
-            return static_cast< const XDataNoConst& >( *fDataObjects.at(typeid(XDataNoConst)) );
+            THROW_EXCEPT_HERE( DataFrameException() << "Data type <" << scarab::type<XDataNoConst>() << "> is not present when const Get() was called" );
         }
-        THROW_EXCEPT_HERE( DataFrameException() << "Data type <" << scarab::type(XDataNoConst()) << "> is not present when const Get() was called" );
+        return static_cast< const XDataNoConst& >( *iter->fDataPtr );
     }
 
     template< typename XData >
     void DataFrame::Set( XData* ptr )
     {
+        // Note: takes ownership of ptr
         typedef std::remove_const_t< XData > XDataNoConst;
-        fDataObjects[ typeid(XDataNoConst) ].reset( ptr );  // take ownership of ptr
+        DoSet( IndexableData::Create<XDataNoConst>( ptr ) );
         return;
     }
 
     template< typename XData >
     void DataFrame::Set( std::unique_ptr< XData >&& ptr )
     {
+        // Note: takes ownership of object pointed to by ptr
         typedef std::remove_const_t< XData > XDataNoConst;
-        fDataObjects[ typeid(XDataNoConst) ] = std::move(ptr);  // take ownership of ptr
+        DoSet( IndexableData::Create<XDataNoConst>( std::move(ptr) ) );
         return;
     }
 
     template< typename XData >
     void DataFrame::Set( const XData& obj )
     {
+        // Note: makes a copy of obj and takes ownership of the copy
         typedef std::remove_const_t< XData > XDataNoConst;
-        fDataObjects[ typeid(XDataNoConst) ].reset( new XDataNoConst(obj) );  // make a copy of obj
+        DoSet( IndexableData::Create<XDataNoConst>( obj ) );
         return;
     }
 
     template< typename XData >
     void DataFrame::Remove()
     {
         typedef std::remove_const_t< XData > XDataNoConst;
-        fDataObjects.erase( typeid(XDataNoConst) );
+        fDataMapByType.erase( typeid(XDataNoConst) );
         return;
     }
 

Python/Bindings/CMakeLists.txt

@@ -16,6 +16,7 @@ include_directories( BEFORE
 set( PYBINDING_HEADERFILES
     Data/DataPybind.hh
     Processor/ProcessorPybind.hh
+    Processor/ProcessorToolboxPybind.hh
     Processor/ProcessorRegistrar.hh
     Processor/ProcessorToolboxPybind.hh
     Processor/PyProcCreatorPybind.hh