diff --git a/CMakeLists.txt b/CMakeLists.txt index 5232858c..2df612b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.19) -project(pcms VERSION 0.1.2 LANGUAGES CXX) +project(pcms VERSION 0.2.0 LANGUAGES CXX) include(GNUInstallDirs) include(CMakePackageConfigHelpers) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c20891d4..acfa75d5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -18,6 +18,7 @@ set(PCMS_HEADERS pcms/profile.h pcms/print.h pcms/partition.h + pcms/coupler.h ) set(PCMS_SOURCES @@ -43,12 +44,6 @@ if (PCMS_ENABLE_OMEGA_H) pcms/uniform_grid.h pcms/point_search.h) endif () -if (PCMS_ENABLE_SERVER) - if (NOT PCMS_ENABLE_OMEGA_H) - message(ERROR "PCMS_ENABLE_OMEGA_H is required for server implementation") - endif () - list(APPEND PCMS_HEADERS pcms/server.h) -endif () find_package(Kokkos REQUIRED) find_package(perfstubs REQUIRED) @@ -67,10 +62,6 @@ endif() if(PCMS_ENABLE_SERVER) target_compile_definitions(pcms_core PUBLIC -DPCMS_HAS_SERVER) endif() -if(PCMS_ENABLE_CLIENT) - list(APPEND PCMS_HEADERS pcms/client.h) - target_compile_definitions(pcms_core PUBLIC -DPCMS_HAS_CLIENT) -endif() if(PCMS_HAS_ASAN) target_compile_options(pcms_core PRIVATE -fsanitize=address -fno-omit-frame-pointer) diff --git a/src/pcms.h b/src/pcms.h index 3fed6ca7..e39c3612 100644 --- a/src/pcms.h +++ b/src/pcms.h @@ -1,11 +1,10 @@ #ifndef PCMS_H_ #define PCMS_H_ -#ifdef PCMS_HAS_SERVER -#include "pcms/server.h" -#endif -#ifdef PCMS_HAS_CLIENT -#include "pcms/client.h" -#endif +#include "pcms/common.h" +#include "pcms/field_communicator.h" +#include "pcms/omega_h_field.h" +#include "pcms/profile.h" +#include "pcms/coupler.h" #endif diff --git a/src/pcms/capi/client.cpp b/src/pcms/capi/client.cpp index 03017b8d..85d59ce0 100644 --- a/src/pcms/capi/client.cpp +++ b/src/pcms/capi/client.cpp @@ -29,13 +29,17 @@ using FieldAdapterVariant = [[nodiscard]] PcmsClientHandle pcms_create_client(const char* name, MPI_Comm comm) { - auto* client = new pcms::CouplerClient(name, comm); - return {reinterpret_cast(client)}; + auto* coupler = new pcms::Coupler(name, comm, false, {}); + auto* app = coupler->AddApplication(name); + PcmsClientHandle handle; + handle.couplerPointer = reinterpret_cast(coupler); + handle.appPointer = reinterpret_cast(app); + return handle; } void pcms_destroy_client(PcmsClientHandle client) { - if (client.pointer != nullptr) - delete reinterpret_cast(client.pointer); + if (client.couplerPointer != nullptr) + delete reinterpret_cast(client.couplerPointer); } PcmsReverseClassificationHandle pcms_load_reverse_classification( const char* file, MPI_Comm comm) @@ -52,9 +56,9 @@ void pcms_destroy_reverse_classification(PcmsReverseClassificationHandle rc) } struct AddFieldVariantOperators { - AddFieldVariantOperators(const char* name, pcms::CouplerClient* client, + AddFieldVariantOperators(const char* name, pcms::Application* app, int participates) - : name_(name), client_(client), participates_(participates) + : name_(name), app_(app), participates_(participates) { } @@ -68,11 +72,11 @@ struct AddFieldVariantOperators pcms::CoupledField* operator()( const FieldAdapter& field_adapter) const noexcept { - return client_->AddField(name_, field_adapter, participates_); + return app_->AddField(name_, field_adapter, participates_); } const char* name_; - pcms::CouplerClient* client_; + pcms::Application* app_; bool participates_; }; @@ -83,8 +87,8 @@ PcmsFieldHandle pcms_add_field(PcmsClientHandle client_handle, const char* name, auto* adapter = reinterpret_cast(adapter_handle.pointer); - auto* client = reinterpret_cast(client_handle.pointer); - PCMS_ALWAYS_ASSERT(client != nullptr); + auto* app = reinterpret_cast(client_handle.appPointer); + PCMS_ALWAYS_ASSERT(app != nullptr); PCMS_ALWAYS_ASSERT(adapter != nullptr); // pcms::CoupledField* field = std::visit( // redev::overloaded{ @@ -94,20 +98,20 @@ PcmsFieldHandle pcms_add_field(PcmsClientHandle client_handle, const char* name, // }}, // *adapter); pcms::CoupledField* field = - std::visit(AddFieldVariantOperators{name, client, participates}, *adapter); + std::visit(AddFieldVariantOperators{name, app, participates}, *adapter); return {reinterpret_cast(field)}; } void pcms_send_field_name(PcmsClientHandle client_handle, const char* name) { - auto* client = reinterpret_cast(client_handle.pointer); - PCMS_ALWAYS_ASSERT(client != nullptr); - client->SendField(name); + auto* app = reinterpret_cast(client_handle.appPointer); + PCMS_ALWAYS_ASSERT(app != nullptr); + app->SendField(name); } void pcms_receive_field_name(PcmsClientHandle client_handle, const char* name) { - auto* client = reinterpret_cast(client_handle.pointer); - PCMS_ALWAYS_ASSERT(client != nullptr); - client->ReceiveField(name); + auto* app = reinterpret_cast(client_handle.appPointer); + PCMS_ALWAYS_ASSERT(app != nullptr); + app->ReceiveField(name); } void pcms_send_field(PcmsFieldHandle field_handle) { @@ -198,25 +202,25 @@ int pcms_reverse_classification_count_verts(PcmsReverseClassificationHandle rc) } void pcms_begin_send_phase(PcmsClientHandle h) { - auto* client = reinterpret_cast(h.pointer); - PCMS_ALWAYS_ASSERT(client != nullptr); - client->BeginSendPhase(); + auto* app = reinterpret_cast(h.appPointer); + PCMS_ALWAYS_ASSERT(app != nullptr); + app->BeginSendPhase(); } void pcms_end_send_phase(PcmsClientHandle h) { - auto* client = reinterpret_cast(h.pointer); - PCMS_ALWAYS_ASSERT(client != nullptr); - client->EndSendPhase(); + auto* app = reinterpret_cast(h.appPointer); + PCMS_ALWAYS_ASSERT(app != nullptr); + app->EndSendPhase(); } void pcms_begin_receive_phase(PcmsClientHandle h) { - auto* client = reinterpret_cast(h.pointer); - PCMS_ALWAYS_ASSERT(client != nullptr); - client->BeginReceivePhase(); + auto* app = reinterpret_cast(h.appPointer); + PCMS_ALWAYS_ASSERT(app != nullptr); + app->BeginReceivePhase(); } void pcms_end_receive_phase(PcmsClientHandle h) { - auto* client = reinterpret_cast(h.pointer); - PCMS_ALWAYS_ASSERT(client != nullptr); - client->EndReceivePhase(); + auto* app = reinterpret_cast(h.appPointer); + PCMS_ALWAYS_ASSERT(app != nullptr); + app->EndReceivePhase(); } \ No newline at end of file diff --git a/src/pcms/capi/client.h b/src/pcms/capi/client.h index e03a3332..5aa8eec6 100644 --- a/src/pcms/capi/client.h +++ b/src/pcms/capi/client.h @@ -9,7 +9,8 @@ extern "C" { struct PcmsClientHandle { - void* pointer; + void* couplerPointer; + void* appPointer; }; typedef struct PcmsClientHandle PcmsClientHandle; struct PcmsOmegaHMeshHandle diff --git a/src/pcms/client.h b/src/pcms/client.h deleted file mode 100644 index a6e4d256..00000000 --- a/src/pcms/client.h +++ /dev/null @@ -1,193 +0,0 @@ -#ifndef PCMS_COUPLING_CLIENT_H -#define PCMS_COUPLING_CLIENT_H -#include "pcms/common.h" -#include "pcms/field_communicator.h" -#include "pcms/profile.h" - -namespace pcms -{ - -class CoupledField -{ -public: - template - CoupledField(const std::string& name, FieldAdapterT field_adapter, - MPI_Comm mpi_comm, redev::Redev& redev, redev::Channel& channel, - bool participates) - { - PCMS_FUNCTION_TIMER; - MPI_Comm mpi_comm_subset = MPI_COMM_NULL; - PCMS_ALWAYS_ASSERT((mpi_comm == MPI_COMM_NULL) ? (participates == false) - : true); - if (mpi_comm != MPI_COMM_NULL) { - int rank = -1; - MPI_Comm_rank(mpi_comm, &rank); - MPI_Comm_split(mpi_comm, participates ? 0 : MPI_UNDEFINED, rank, - &mpi_comm_subset); - } - coupled_field_ = - std::make_unique>( - name, std::move(field_adapter), mpi_comm_subset, redev, channel, - participates); - } - - void Send(Mode mode = Mode::Synchronous) - { - PCMS_FUNCTION_TIMER; - coupled_field_->Send(mode); - } - void Receive() - { - PCMS_FUNCTION_TIMER; - coupled_field_->Receive(); - } - struct CoupledFieldConcept - { - virtual void Send(Mode) = 0; - virtual void Receive() = 0; - virtual ~CoupledFieldConcept() = default; - }; - template - struct CoupledFieldModel final : CoupledFieldConcept - { - using value_type = typename FieldAdapterT::value_type; - - CoupledFieldModel(const std::string& name, FieldAdapterT&& field_adapter, - MPI_Comm mpi_comm_subset, redev::Redev& redev, - redev::Channel& channel, bool participates) - : mpi_comm_subset_(mpi_comm_subset), - field_adapter_(std::move(field_adapter)), - comm_(FieldCommunicator(name, mpi_comm_subset_, redev, channel, - field_adapter_)) - { - PCMS_FUNCTION_TIMER; - } - void Send(Mode mode) final - { - PCMS_FUNCTION_TIMER; - comm_.Send(mode); - }; - void Receive() final - { - PCMS_FUNCTION_TIMER; - comm_.Receive(); - }; - ~CoupledFieldModel() - { - PCMS_FUNCTION_TIMER; - if (mpi_comm_subset_ != MPI_COMM_NULL) - MPI_Comm_free(&mpi_comm_subset_); - } - - MPI_Comm mpi_comm_subset_; - FieldAdapterT field_adapter_; - FieldCommunicator comm_; - }; - -private: - std::unique_ptr coupled_field_; -}; - -class CouplerClient -{ -public: - CouplerClient(std::string name, MPI_Comm comm, - redev::TransportType transport_type = redev::TransportType::BP4, - adios2::Params params = {{"Streaming", "On"}, - {"OpenTimeoutSecs", "400"}}, - std::string path = "") - : name_(std::move(name)), - mpi_comm_(comm), - redev_(comm), - channel_{redev_.CreateAdiosChannel(name_, std::move(params), - transport_type, std::move(path))} - { - PCMS_FUNCTION_TIMER; - } - - [[nodiscard]] const redev::Partition& GetPartition() const - { - PCMS_FUNCTION_TIMER; - return redev_.GetPartition(); - } - - /** - * @known_issue - * The redev partion has to be same as the partition of the OH mesh. - * It asserts the number of elements sent and received are same. - * otherwise, ConstructPermutation() will fail. - */ - template - CoupledField* AddField(std::string name, FieldAdapterT field_adapter, - bool participates = true) - { - PCMS_FUNCTION_TIMER; - auto [it, inserted] = - fields_.template try_emplace(name, name, std::move(field_adapter), - mpi_comm_, redev_, channel_, participates); - if (!inserted) { - std::cerr << "OHField with this name" << name << "already exists!\n"; - std::terminate(); - } - return &(it->second); - } - - // take a string& since map cannot be searched with string_view - // (heterogeneous lookup) - void SendField(const std::string& name, Mode mode = Mode::Synchronous) - { - PCMS_FUNCTION_TIMER; - PCMS_ALWAYS_ASSERT(InSendPhase()); - detail::find_or_error(name, fields_).Send(mode); - }; - // take a string& since map cannot be searched with string_view - // (heterogeneous lookup) - void ReceiveField(const std::string& name) - { - PCMS_FUNCTION_TIMER; - PCMS_ALWAYS_ASSERT(InReceivePhase()); - detail::find_or_error(name, fields_).Receive(); - }; - [[nodiscard]] bool InSendPhase() const noexcept - { - PCMS_FUNCTION_TIMER; - return channel_.InSendCommunicationPhase(); - } - [[nodiscard]] bool InReceivePhase() const noexcept - { - PCMS_FUNCTION_TIMER; - return channel_.InReceiveCommunicationPhase(); - } - void BeginSendPhase() - { - PCMS_FUNCTION_TIMER; - channel_.BeginSendCommunicationPhase(); - } - void EndSendPhase() - { - PCMS_FUNCTION_TIMER; - channel_.EndSendCommunicationPhase(); - } - void BeginReceivePhase() - { - PCMS_FUNCTION_TIMER; - channel_.BeginReceiveCommunicationPhase(); - } - void EndReceivePhase() - { - PCMS_FUNCTION_TIMER; - channel_.EndReceiveCommunicationPhase(); - } - -private: - std::string name_; - MPI_Comm mpi_comm_; - redev::Redev redev_; - // map rather than unordered_map is necessary to avoid iterator invalidation. - // This is important because we pass pointers to the fields out of this class - std::map fields_; - redev::Channel channel_; -}; -} // namespace pcms - -#endif // PCMS_COUPLING_CLIENT_H diff --git a/src/pcms/server.h b/src/pcms/coupler.h similarity index 53% rename from src/pcms/server.h rename to src/pcms/coupler.h index eb74d50e..45b0db2f 100644 --- a/src/pcms/server.h +++ b/src/pcms/coupler.h @@ -1,45 +1,34 @@ -#ifndef PCMS_COUPLING_SERVER_H -#define PCMS_COUPLING_SERVER_H +#ifndef PCMS_COUPLER_H +#define PCMS_COUPLER_H #include "pcms/common.h" #include "pcms/field_communicator.h" -#include "pcms/omega_h_field.h" #include "pcms/profile.h" -#include -#include namespace pcms { -// TODO: come up with better name for this...Don't like CoupledFieldServer -// because it's necessarily tied to the Server of the xgc_coupler -class ConvertibleCoupledField + +class CoupledField { public: - template - ConvertibleCoupledField(const std::string& name, FieldAdapterT field_adapter, - FieldCommunicator field_comm, - Omega_h::Mesh& internal_mesh, - Omega_h::Read internal_field_mask = {}) - : internal_field_{OmegaHField( - name + ".__internal__", internal_mesh, internal_field_mask, "", 10, 10, field_adapter.GetEntityType())} - { - PCMS_FUNCTION_TIMER; - coupled_field_ = std::make_unique>( - std::move(field_adapter), std::move(field_comm)); - } template - ConvertibleCoupledField(const std::string& name, FieldAdapterT field_adapter, - MPI_Comm mpi_comm, redev::Redev& redev, - redev::Channel& channel, Omega_h::Mesh& internal_mesh, - Omega_h::Read internal_field_mask) - : internal_field_{OmegaHField( - name + ".__internal__", internal_mesh, internal_field_mask, "", 10, 10, field_adapter.GetEntityType())} + CoupledField(const std::string& name, FieldAdapterT field_adapter, + MPI_Comm mpi_comm, redev::Redev& redev, redev::Channel& channel, + bool participates = true) { PCMS_FUNCTION_TIMER; + MPI_Comm mpi_comm_subset = MPI_COMM_NULL; + PCMS_ALWAYS_ASSERT((mpi_comm == MPI_COMM_NULL) ? (participates == false) + : true); + if (mpi_comm != MPI_COMM_NULL) { + int rank = -1; + MPI_Comm_rank(mpi_comm, &rank); + MPI_Comm_split(mpi_comm, participates ? 0 : MPI_UNDEFINED, rank, + &mpi_comm_subset); + } coupled_field_ = std::make_unique>( - name, std::move(field_adapter), mpi_comm, redev, channel); + name, std::move(field_adapter), mpi_comm_subset, redev, channel, + participates); } void Send(Mode mode = Mode::Synchronous) @@ -47,21 +36,11 @@ class ConvertibleCoupledField PCMS_FUNCTION_TIMER; coupled_field_->Send(mode); } - void Receive(Mode mode=Mode::Synchronous) + void Receive(Mode mode = Mode::Synchronous) { PCMS_FUNCTION_TIMER; coupled_field_->Receive(mode); } - [[nodiscard]] InternalField& GetInternalField() noexcept - { - PCMS_FUNCTION_TIMER; - return internal_field_; - } - [[nodiscard]] const InternalField& GetInternalField() const noexcept - { - PCMS_FUNCTION_TIMER; - return internal_field_; - } template [[nodiscard]] T* GetFieldAdapter() const { @@ -87,20 +66,13 @@ class ConvertibleCoupledField { using value_type = typename FieldAdapterT::value_type; - CoupledFieldModel(FieldAdapterT&& field_adapter, - FieldCommunicator&& comm) - : field_adapter_(std::move(field_adapter)), - comm_(std::move(comm)), - type_info_(typeid(FieldAdapterT)) - { - PCMS_FUNCTION_TIMER; - } CoupledFieldModel(const std::string& name, FieldAdapterT&& field_adapter, - MPI_Comm mpi_comm, redev::Redev& redev, - redev::Channel& channel) - : field_adapter_(std::move(field_adapter)), - comm_(FieldCommunicator(name, mpi_comm, redev, channel, - field_adapter_)), + MPI_Comm mpi_comm_subset, redev::Redev& redev, + redev::Channel& channel, bool participates) + : mpi_comm_subset_(mpi_comm_subset), + field_adapter_(std::move(field_adapter)), + comm_(FieldCommunicator(name, mpi_comm_subset_, redev, channel, + field_adapter_)), type_info_(typeid(FieldAdapterT)) { PCMS_FUNCTION_TIMER; @@ -123,7 +95,14 @@ class ConvertibleCoupledField { return reinterpret_cast(&field_adapter_); }; + ~CoupledFieldModel() + { + PCMS_FUNCTION_TIMER; + if (mpi_comm_subset_ != MPI_COMM_NULL) + MPI_Comm_free(&mpi_comm_subset_); + } + MPI_Comm mpi_comm_subset_; FieldAdapterT field_adapter_; FieldCommunicator comm_; const std::type_info& type_info_; @@ -131,42 +110,34 @@ class ConvertibleCoupledField private: std::unique_ptr coupled_field_; - // even though we know the type of the internal field, - // we store it as the InternalField variant since this avoids any copies - // This comes at the cost of a slightly larger type with need to use the get<> - // function - InternalField internal_field_; }; -// TODO: strategy to merge Server/CLient Application and Fields + class Application { public: - Application(std::string name, redev::Redev& rdv, MPI_Comm comm, - redev::Redev& redev, Omega_h::Mesh& internal_mesh, - adios2::Params params, redev::TransportType transport_type, + Application(std::string name, MPI_Comm comm, + redev::Redev& redev, adios2::Params params, + redev::TransportType transport_type, std::string path) : mpi_comm_(comm), redev_(redev), - channel_{rdv.CreateAdiosChannel(std::move(name), std::move(params), - transport_type, std::move(path))}, - internal_mesh_{internal_mesh} + channel_{redev_.CreateAdiosChannel(std::move(name), std::move(params), + transport_type, std::move(path))} { PCMS_FUNCTION_TIMER; } // FIXME should take a file path for the parameters, not take adios2 params. // These fields are supposed to be agnostic to adios2... template - ConvertibleCoupledField* AddField( - std::string name, FieldAdapterT&& field_adapter, - Omega_h::Read internal_field_mask = {}) + CoupledField* AddField( + std::string name, FieldAdapterT&& field_adapter, bool participates = true) { PCMS_FUNCTION_TIMER; auto [it, inserted] = fields_.template try_emplace( name, name, std::forward(field_adapter), mpi_comm_, redev_, - channel_, internal_mesh_, - internal_field_mask); + channel_, participates); if (!inserted) { - std::cerr << "OHField with this name" << name << "already exists!\n"; + std::cerr << "Field with this name" << name << "already exists!\n"; std::terminate(); } return &(it->second); @@ -234,31 +205,35 @@ class Application // map is used rather than unordered_map because we give pointers to the // internal data and rehash of unordered_map can cause pointer invalidation. // map is less cache friendly, but pointers are not invalidated. - std::map fields_; - Omega_h::Mesh& internal_mesh_; + std::map fields_; }; -class CouplerServer +class Coupler { +private: + redev::Redev SetUpRedev(bool isServer, redev::Partition partition) { + if (isServer) + return redev::Redev(mpi_comm_, std::move(partition), ProcessType::Server); + else + return redev::Redev(mpi_comm_); + } public: - CouplerServer(std::string name, MPI_Comm comm, redev::Partition partition, - Omega_h::Mesh& mesh) + Coupler(std::string name, MPI_Comm comm, bool isServer, redev::Partition partition) : name_(std::move(name)), mpi_comm_(comm), - redev_({comm, std::move(partition), ProcessType::Server}), - internal_mesh_(mesh) + redev_(SetUpRedev(isServer, std::move(partition))) { PCMS_FUNCTION_TIMER; } Application* AddApplication( std::string name, std::string path = "", redev::TransportType transport_type = redev::TransportType::BP4, - adios2::Params params = {{"Streaming", "On"}, {"OpenTimeoutSecs", "400"}}) + adios2::Params params = {{"Streaming", "On"}, {"OpenTimeoutSecs", "60"}}) { PCMS_FUNCTION_TIMER; auto key = path + name; auto [it, inserted] = applications_.template try_emplace( - key, std::move(name), redev_, mpi_comm_, redev_, internal_mesh_, + key, std::move(name), mpi_comm_, redev_, std::move(params), transport_type, std::move(path)); if (!inserted) { std::cerr << "Application with name " << name << "already exists!\n"; @@ -271,30 +246,15 @@ class CouplerServer { return redev_.GetPartition(); } - [[nodiscard]] Omega_h::Mesh& GetMesh() noexcept { return internal_mesh_; } - - [[nodiscard]] const auto& GetInternalFields() const noexcept - { - return internal_fields_; - } - - // TODO: consider an "advanced api" wrapper of some sort and protect direct - // access to the internal fields with passkey idom or some other way. User - // could get unexpected behavior if they mess with the internal field map. - /// This function should not be used directly it is experimental for Philip - /// to do Benesh development. Expect that it will be removed in the future. - [[nodiscard]] auto& GetInternalFields() noexcept { return internal_fields_; } private: std::string name_; MPI_Comm mpi_comm_; redev::Redev redev_; - // xgc_coupler owns internal fields since both gather/scatter ops use these - // these internal fields correspond to the "Combined" fields - std::map internal_fields_; // gather and scatter operations have reference to internal fields std::map applications_; - Omega_h::Mesh& internal_mesh_; }; + } // namespace pcms -#endif // PCMS_COUPLING_SERVER_H + +#endif // PCMS_COUPLER_H diff --git a/src/pcms/fortranapi/client_wrap.c b/src/pcms/fortranapi/client_wrap.c index 9b6275a8..f92bf680 100644 --- a/src/pcms/fortranapi/client_wrap.c +++ b/src/pcms/fortranapi/client_wrap.c @@ -277,7 +277,7 @@ SWIGEXPORT void _wrap_PcmsClientHandle_pointer_set(SwigClassWrapper* farg1, arg1 = (struct PcmsClientHandle*)farg1->cptr; arg2 = (void*)(*farg2); if (arg1) - (arg1)->pointer = arg2; + (arg1)->appPointer = arg2; } SWIGEXPORT void* _wrap_PcmsClientHandle_pointer_get(SwigClassWrapper* farg1) @@ -289,7 +289,7 @@ SWIGEXPORT void* _wrap_PcmsClientHandle_pointer_get(SwigClassWrapper* farg1) SWIG_check_nonnull(farg1->cptr, "struct PcmsClientHandle *", "PcmsClientHandle", "PcmsClientHandle::pointer", return 0); arg1 = (struct PcmsClientHandle*)farg1->cptr; - result = (void*)((arg1)->pointer); + result = (void*)((arg1)->appPointer); fresult = (void*)(result); return fresult; } diff --git a/test/field_transfer_example.cpp b/test/field_transfer_example.cpp index c0fade6c..8664b5cb 100644 --- a/test/field_transfer_example.cpp +++ b/test/field_transfer_example.cpp @@ -41,7 +41,7 @@ struct MeanCombiner void SetApplicationFields(const OHField& app_a_field, const OHField& app_b_field); -using pcms::ConvertibleCoupledField; +using pcms::CoupledField; using pcms::ProcessType; using pcms::FieldCommunicator; using pcms::InternalField; diff --git a/test/test_proxy_coupling.cpp b/test/test_proxy_coupling.cpp index 43133341..5aead8f0 100644 --- a/test/test_proxy_coupling.cpp +++ b/test/test_proxy_coupling.cpp @@ -11,8 +11,6 @@ #include using pcms::Copy; -using pcms::CouplerClient; -using pcms::CouplerServer; using pcms::GO; using pcms::Lagrange; using pcms::make_array_view; @@ -27,40 +25,42 @@ namespace ts = test_support; void xgc_delta_f(MPI_Comm comm, Omega_h::Mesh& mesh) { - CouplerClient cpl("proxy_couple_xgc_delta_f", comm); + pcms::Coupler coupler("proxy_couple", comm, false, {}); + pcms::Application* app = coupler.AddApplication("proxy_couple_xgc_delta_f"); auto is_overlap = ts::markOverlapMeshEntities(mesh, ts::IsModelEntInOverlap{}); - cpl.AddField("gids", + app->AddField("gids", OmegaHFieldAdapter("global", mesh, is_overlap)); - cpl.AddField("gids2", + app->AddField("gids2", OmegaHFieldAdapter("global", mesh, is_overlap)); do { for (int i = 0; i < COMM_ROUNDS; ++i) { - cpl.BeginSendPhase(); - cpl.SendField("gids"); //(Alt) df_gid_field->Send(); - cpl.SendField("gids2"); //(Alt) df_gid_field->Send(); - cpl.EndSendPhase(); - cpl.BeginReceivePhase(); - cpl.ReceiveField("gids"); //(Alt) df_gid_field->Receive(); - cpl.EndReceivePhase(); + app->BeginSendPhase(); + app->SendField("gids"); //(Alt) df_gid_field->Send(); + app->SendField("gids2"); //(Alt) df_gid_field->Send(); + app->EndSendPhase(); + app->BeginReceivePhase(); + app->ReceiveField("gids"); //(Alt) df_gid_field->Receive(); + app->EndReceivePhase(); // cpl.ReceiveField("gids2"); //(Alt) df_gid_field->Receive(); } } while (!done); } void xgc_total_f(MPI_Comm comm, Omega_h::Mesh& mesh) { - pcms::CouplerClient cpl("proxy_couple_xgc_total_f", comm); + pcms::Coupler coupler("proxy_couple", comm, false, {}); + pcms::Application* app = coupler.AddApplication("proxy_couple_xgc_total_f"); auto is_overlap = ts::markOverlapMeshEntities(mesh, ts::IsModelEntInOverlap{}); - cpl.AddField("gids", + app->AddField("gids", OmegaHFieldAdapter("global", mesh, is_overlap)); do { for (int i = 0; i < COMM_ROUNDS; ++i) { - cpl.BeginSendPhase(); - cpl.SendField("gids"); //(Alt) tf_gid_field->Send(); - cpl.EndSendPhase(); - cpl.BeginReceivePhase(); - cpl.ReceiveField("gids"); //(Alt) tf_gid_field->Receive(); - cpl.EndReceivePhase(); + app->BeginSendPhase(); + app->SendField("gids"); //(Alt) tf_gid_field->Send(); + app->EndSendPhase(); + app->BeginReceivePhase(); + app->ReceiveField("gids"); //(Alt) tf_gid_field->Receive(); + app->EndReceivePhase(); } } while (!done); } @@ -69,9 +69,9 @@ void xgc_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, std::string_view cpn_file) // coupling server using same mesh as application // note the xgc_coupler stores a reference to the internal mesh and it is the // user responsibility to keep it alive! - pcms::CouplerServer cpl( - "proxy_couple", comm, - redev::Partition{ts::setupServerPartition(mesh, cpn_file)}, mesh); + pcms::Coupler cpl( + "proxy_couple", comm, true, + redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); const auto partition = std::get(cpl.GetPartition()); auto is_overlap = ts::markServerOverlapRegion(mesh, partition, ts::IsModelEntInOverlap{}); @@ -79,14 +79,11 @@ void xgc_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, std::string_view cpn_file) auto* delta_f = cpl.AddApplication("proxy_couple_xgc_delta_f"); // TODO, fields should have a transfer policy rather than parameters auto* total_f_gids = total_f->AddField( - "gids", OmegaHFieldAdapter("total_f_gids", mesh, is_overlap), - is_overlap); + "gids", OmegaHFieldAdapter("total_f_gids", mesh, is_overlap)); auto* delta_f_gids = delta_f->AddField( - "gids", OmegaHFieldAdapter("delta_f_gids", mesh, is_overlap), - is_overlap); + "gids", OmegaHFieldAdapter("delta_f_gids", mesh, is_overlap)); auto* delta_f_gids2 = delta_f->AddField( - "gids2", OmegaHFieldAdapter("delta_f_gids2", mesh, is_overlap), - is_overlap); + "gids2", OmegaHFieldAdapter("delta_f_gids2", mesh, is_overlap)); do { for (int i = 0; i < COMM_ROUNDS; ++i) { total_f->ReceivePhase([&]() { total_f_gids->Receive(); }); diff --git a/test/test_proxy_coupling_xgc_server.cpp b/test/test_proxy_coupling_xgc_server.cpp index 086652f0..69039348 100644 --- a/test/test_proxy_coupling_xgc_server.cpp +++ b/test/test_proxy_coupling_xgc_server.cpp @@ -9,8 +9,6 @@ using pcms::ConstructRCFromOmegaHMesh; using pcms::Copy; -using pcms::CouplerClient; -using pcms::CouplerServer; using pcms::GO; using pcms::Lagrange; using pcms::make_array_view; @@ -27,9 +25,9 @@ void xgc_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, std::string_view cpn_file) // coupling server using same mesh as application // note the xgc_coupler stores a reference to the internal mesh and it is the // user responsibility to keep it alive! - pcms::CouplerServer cpl( - "proxy_couple_server", comm, - redev::Partition{ts::setupServerPartition(mesh, cpn_file)}, mesh); + pcms::Coupler cpl( + "proxy_couple_server", comm, true, + redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); const auto partition = std::get(cpl.GetPartition()); ReverseClassificationVertex rc; if (mesh.has_tag(0, "simNumbering")) { @@ -44,7 +42,7 @@ void xgc_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, std::string_view cpn_file) constexpr int nplanes = 2; std::array, nplanes> data; - std::vector fields; + std::vector fields; for (int i = 0; i < nplanes; ++i) { data[i].resize(mesh.nverts()); std::stringstream ss; @@ -52,25 +50,25 @@ void xgc_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, std::string_view cpn_file) auto field_adapter = pcms::XGCFieldAdapter( ss.str(), comm, make_array_view(data[i]), rc, ts::IsModelEntInOverlap{}); fields.push_back( - application->AddField(ss.str(), std::move(field_adapter), is_overlap)); + application->AddField(ss.str(), std::move(field_adapter))); } do { application->ReceivePhase([&]() { std::for_each(fields.begin(), fields.end(), - [](pcms::ConvertibleCoupledField* f) { f->Receive(); }); + [](pcms::CoupledField* f) { f->Receive(); }); }); application->SendPhase([&]() { std::for_each(fields.begin(), fields.end(), - [](pcms::ConvertibleCoupledField* f) { f->Send(); }); + [](pcms::CoupledField* f) { f->Send(); }); }); application->ReceivePhase([&]() { std::for_each(fields.begin(), fields.end(), - [](pcms::ConvertibleCoupledField* f) { f->Receive(); }); + [](pcms::CoupledField* f) { f->Receive(); }); }); application->SendPhase([&]() { std::for_each(fields.begin(), fields.end(), - [](pcms::ConvertibleCoupledField* f) { f->Send(); }); + [](pcms::CoupledField* f) { f->Send(); }); }); } while (!done); @@ -82,9 +80,9 @@ void omegah_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, // coupling server using same mesh as application // note the xgc_coupler stores a reference to the internal mesh and it is the // user responsibility to keep it alive! - pcms::CouplerServer cpl( - "proxy_couple_server", comm, - redev::Partition{ts::setupServerPartition(mesh, cpn_file)}, mesh); + pcms::Coupler cpl( + "proxy_couple_server", comm, true, + redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); const auto partition = std::get(cpl.GetPartition()); auto* application = cpl.AddApplication("proxy_couple"); std::string numbering; @@ -102,31 +100,31 @@ void omegah_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, auto is_overlap = ts::markServerOverlapRegion(mesh, partition, ts::IsModelEntInOverlap{}); constexpr int nplanes = 2; - std::vector fields; + std::vector fields; for (int i = 0; i < nplanes; ++i) { std::stringstream ss; ss << "xgc_gids_plane_" << i; auto field_adapter = pcms::OmegaHFieldAdapter(ss.str(), mesh, is_overlap, numbering); fields.push_back( - application->AddField(ss.str(), std::move(field_adapter), is_overlap)); + application->AddField(ss.str(), std::move(field_adapter))); } do { application->ReceivePhase([&]() { std::for_each(fields.begin(), fields.end(), - [](pcms::ConvertibleCoupledField* f) { f->Receive(); }); + [](pcms::CoupledField* f) { f->Receive(); }); }); application->SendPhase([&]() { std::for_each(fields.begin(), fields.end(), - [](pcms::ConvertibleCoupledField* f) { f->Send(); }); + [](pcms::CoupledField* f) { f->Send(); }); }); application->ReceivePhase([&]() { std::for_each(fields.begin(), fields.end(), - [](pcms::ConvertibleCoupledField* f) { f->Receive(); }); + [](pcms::CoupledField* f) { f->Receive(); }); }); application->SendPhase([&]() { std::for_each(fields.begin(), fields.end(), - [](pcms::ConvertibleCoupledField* f) { f->Send(); }); + [](pcms::CoupledField* f) { f->Send(); }); }); } while (!done); Omega_h::vtk::write_parallel("proxy_couple", &mesh, mesh.dim()); diff --git a/test/xgc_n0_coupling_server.cpp b/test/xgc_n0_coupling_server.cpp index 68490f41..2b1d1853 100644 --- a/test/xgc_n0_coupling_server.cpp +++ b/test/xgc_n0_coupling_server.cpp @@ -8,8 +8,6 @@ #include using pcms::Copy; -using pcms::CouplerClient; -using pcms::CouplerServer; using pcms::GO; using pcms::LO; using pcms::OmegaHFieldAdapter; @@ -22,7 +20,7 @@ namespace ts = test_support; // [[nodiscard]] -static pcms::ConvertibleCoupledField* AddField(pcms::Application *application, const std::string& name, const std::string& path, Omega_h::Read is_overlap, const std::string& numbering, Omega_h::Mesh& mesh, int plane) { +static pcms::CoupledField* AddField(pcms::Application *application, const std::string& name, const std::string& path, Omega_h::Read is_overlap, const std::string& numbering, Omega_h::Mesh& mesh, int plane) { PCMS_ALWAYS_ASSERT(application != nullptr); std::stringstream field_name; field_name << name; @@ -31,32 +29,31 @@ static pcms::ConvertibleCoupledField* AddField(pcms::Application *application, c } return application->AddField(field_name.str(), pcms::OmegaHFieldAdapter( - path+field_name.str(), mesh, is_overlap, numbering), - is_overlap); + path+field_name.str(), mesh, is_overlap, numbering)); } struct XGCAnalysis { - using FieldVec = std::vector; + using FieldVec = std::vector; std::array dpot; FieldVec pot0; std::array edensity; std::array idensity; - pcms::ConvertibleCoupledField* psi; - pcms::ConvertibleCoupledField* gids; + pcms::CoupledField* psi; + pcms::CoupledField* gids; }; -static void ReceiveFields(const std::vector & fields) { +static void ReceiveFields(const std::vector & fields) { for(auto* field : fields) { field->Receive(); } } -static void SendFields(const std::vector & fields) { +static void SendFields(const std::vector & fields) { for(auto* field : fields) { field->Send(); } } -static void CopyFields(const std::vector & from_fields, - const std::vector & to_fields) { +static void CopyFields(const std::vector & from_fields, + const std::vector & to_fields) { PCMS_ALWAYS_ASSERT(from_fields.size() == to_fields.size()); for(size_t i=0; iGetFieldAdapter>(); @@ -81,8 +78,8 @@ static void AverageAndSetField(const pcms::OmegaHField & a, pcms::OmegaHField * Takes the average of each pair of fields and sets the results in the the second * argument */ -static void AverageAndSetFields(const std::vector & from_fields, - const std::vector & to_fields) { +static void AverageAndSetFields(const std::vector & from_fields, + const std::vector & to_fields) { PCMS_ALWAYS_ASSERT(from_fields.size() == to_fields.size()); for(size_t i=0; iGetFieldAdapter>(); @@ -192,8 +189,8 @@ void omegah_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, auto time1 = std::chrono::steady_clock::now(); - pcms::CouplerServer cpl("xgc_n0_coupling", comm, - redev::Partition{ts::setupServerPartition(mesh, cpn_file)}, mesh); + pcms::Coupler cpl("xgc_n0_coupling", comm, true, + redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); const auto partition = std::get(cpl.GetPartition()); std::string numbering = "simNumbering"; PCMS_ALWAYS_ASSERT(mesh.has_tag(0, numbering));