From 6eb73c8d56029d33fb93db8dd90374fba5fb1512 Mon Sep 17 00:00:00 2001 From: Angelyr Date: Wed, 26 Mar 2025 15:51:24 -0400 Subject: [PATCH 01/12] created coupler class --- src/CMakeLists.txt | 1 + src/pcms/client.h | 82 +------------------ src/pcms/coupler.h | 200 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 202 insertions(+), 81 deletions(-) create mode 100644 src/pcms/coupler.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c20891d4..b9d240ef 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -18,6 +18,7 @@ set(PCMS_HEADERS pcms/profile.h pcms/print.h pcms/partition.h + pcms/coupler.h ) set(PCMS_SOURCES diff --git a/src/pcms/client.h b/src/pcms/client.h index a6e4d256..1aa95c22 100644 --- a/src/pcms/client.h +++ b/src/pcms/client.h @@ -3,91 +3,11 @@ #include "pcms/common.h" #include "pcms/field_communicator.h" #include "pcms/profile.h" +#include "pcms/coupler.h" namespace pcms { -class CoupledField -{ -public: - template - CoupledField(const std::string& name, FieldAdapterT field_adapter, - MPI_Comm mpi_comm, redev::Redev& redev, redev::Channel& channel, - bool participates) - { - PCMS_FUNCTION_TIMER; - MPI_Comm mpi_comm_subset = MPI_COMM_NULL; - PCMS_ALWAYS_ASSERT((mpi_comm == MPI_COMM_NULL) ? (participates == false) - : true); - if (mpi_comm != MPI_COMM_NULL) { - int rank = -1; - MPI_Comm_rank(mpi_comm, &rank); - MPI_Comm_split(mpi_comm, participates ? 0 : MPI_UNDEFINED, rank, - &mpi_comm_subset); - } - coupled_field_ = - std::make_unique>( - name, std::move(field_adapter), mpi_comm_subset, redev, channel, - participates); - } - - void Send(Mode mode = Mode::Synchronous) - { - PCMS_FUNCTION_TIMER; - coupled_field_->Send(mode); - } - void Receive() - { - PCMS_FUNCTION_TIMER; - coupled_field_->Receive(); - } - struct CoupledFieldConcept - { - virtual void Send(Mode) = 0; - virtual void Receive() = 0; - virtual ~CoupledFieldConcept() = default; - }; - template - struct CoupledFieldModel final : CoupledFieldConcept - { - using value_type = typename FieldAdapterT::value_type; - - CoupledFieldModel(const std::string& name, FieldAdapterT&& field_adapter, - MPI_Comm mpi_comm_subset, redev::Redev& redev, - redev::Channel& channel, bool participates) - : mpi_comm_subset_(mpi_comm_subset), - field_adapter_(std::move(field_adapter)), - comm_(FieldCommunicator(name, mpi_comm_subset_, redev, channel, - field_adapter_)) - { - PCMS_FUNCTION_TIMER; - } - void Send(Mode mode) final - { - PCMS_FUNCTION_TIMER; - comm_.Send(mode); - }; - void Receive() final - { - PCMS_FUNCTION_TIMER; - comm_.Receive(); - }; - ~CoupledFieldModel() - { - PCMS_FUNCTION_TIMER; - if (mpi_comm_subset_ != MPI_COMM_NULL) - MPI_Comm_free(&mpi_comm_subset_); - } - - MPI_Comm mpi_comm_subset_; - FieldAdapterT field_adapter_; - FieldCommunicator comm_; - }; - -private: - std::unique_ptr coupled_field_; -}; - class CouplerClient { public: diff --git a/src/pcms/coupler.h b/src/pcms/coupler.h new file mode 100644 index 00000000..a57f4ab9 --- /dev/null +++ b/src/pcms/coupler.h @@ -0,0 +1,200 @@ +#ifndef PCMS_COUPLER_H +#define PCMS_COUPLER_H +#include "pcms/common.h" +#include "pcms/field_communicator.h" +#include "pcms/profile.h" + +namespace pcms +{ + +class CoupledField +{ +public: + template + CoupledField(const std::string& name, FieldAdapterT field_adapter, + MPI_Comm mpi_comm, redev::Redev& redev, redev::Channel& channel, + bool participates) + { + PCMS_FUNCTION_TIMER; + MPI_Comm mpi_comm_subset = MPI_COMM_NULL; + PCMS_ALWAYS_ASSERT((mpi_comm == MPI_COMM_NULL) ? (participates == false) + : true); + if (mpi_comm != MPI_COMM_NULL) { + int rank = -1; + MPI_Comm_rank(mpi_comm, &rank); + MPI_Comm_split(mpi_comm, participates ? 0 : MPI_UNDEFINED, rank, + &mpi_comm_subset); + } + coupled_field_ = + std::make_unique>( + name, std::move(field_adapter), mpi_comm_subset, redev, channel, + participates); + } + + void Send(Mode mode = Mode::Synchronous) + { + PCMS_FUNCTION_TIMER; + coupled_field_->Send(mode); + } + void Receive() + { + PCMS_FUNCTION_TIMER; + coupled_field_->Receive(); + } + struct CoupledFieldConcept + { + virtual void Send(Mode) = 0; + virtual void Receive() = 0; + virtual ~CoupledFieldConcept() = default; + }; + template + struct CoupledFieldModel final : CoupledFieldConcept + { + using value_type = typename FieldAdapterT::value_type; + + CoupledFieldModel(const std::string& name, FieldAdapterT&& field_adapter, + MPI_Comm mpi_comm_subset, redev::Redev& redev, + redev::Channel& channel, bool participates) + : mpi_comm_subset_(mpi_comm_subset), + field_adapter_(std::move(field_adapter)), + comm_(FieldCommunicator(name, mpi_comm_subset_, redev, channel, + field_adapter_)) + { + PCMS_FUNCTION_TIMER; + } + void Send(Mode mode) final + { + PCMS_FUNCTION_TIMER; + comm_.Send(mode); + }; + void Receive() final + { + PCMS_FUNCTION_TIMER; + comm_.Receive(); + }; + ~CoupledFieldModel() + { + PCMS_FUNCTION_TIMER; + if (mpi_comm_subset_ != MPI_COMM_NULL) + MPI_Comm_free(&mpi_comm_subset_); + } + + MPI_Comm mpi_comm_subset_; + FieldAdapterT field_adapter_; + FieldCommunicator comm_; + }; + +private: + std::unique_ptr coupled_field_; +}; + +class Coupler +{ +private: + redev::Redev SetUpRedev(bool isRdv, redev::Partition partition) { + if (isRdv) + return redev::Redev(mpi_comm_, std::move(partition)); + else + return redev::Redev(mpi_comm_); + } +public: + Coupler(std::string name, MPI_Comm comm, bool isRdv, redev::Partition partition, + redev::TransportType transport_type = redev::TransportType::BP4, + adios2::Params params = {{"Streaming", "On"}, + {"OpenTimeoutSecs", "60"}}, + std::string path = "") + : name_(std::move(name)), + mpi_comm_(comm), + redev_(SetUpRedev(isRdv, std::move(partition))), + channel_{redev_.CreateAdiosChannel(name_, std::move(params), + transport_type, std::move(path))} + { + PCMS_FUNCTION_TIMER; + } + + [[nodiscard]] const redev::Partition& GetPartition() const + { + PCMS_FUNCTION_TIMER; + return redev_.GetPartition(); + } + + /** + * @known_issue + * The redev partion has to be same as the partition of the OH mesh. + * It asserts the number of elements sent and received are same. + * otherwise, ConstructPermutation() will fail. + */ + template + CoupledField* AddField(std::string name, FieldAdapterT field_adapter, + bool participates = true) + { + PCMS_FUNCTION_TIMER; + auto [it, inserted] = + fields_.template try_emplace(name, name, std::move(field_adapter), + mpi_comm_, redev_, channel_, participates); + if (!inserted) { + std::cerr << "OHField with this name" << name << "already exists!\n"; + std::terminate(); + } + return &(it->second); + } + + // take a string& since map cannot be searched with string_view + // (heterogeneous lookup) + void SendField(const std::string& name, Mode mode = Mode::Synchronous) + { + PCMS_FUNCTION_TIMER; + PCMS_ALWAYS_ASSERT(InSendPhase()); + detail::find_or_error(name, fields_).Send(mode); + }; + // take a string& since map cannot be searched with string_view + // (heterogeneous lookup) + void ReceiveField(const std::string& name) + { + PCMS_FUNCTION_TIMER; + PCMS_ALWAYS_ASSERT(InReceivePhase()); + detail::find_or_error(name, fields_).Receive(); + }; + [[nodiscard]] bool InSendPhase() const noexcept + { + PCMS_FUNCTION_TIMER; + return channel_.InSendCommunicationPhase(); + } + [[nodiscard]] bool InReceivePhase() const noexcept + { + PCMS_FUNCTION_TIMER; + return channel_.InReceiveCommunicationPhase(); + } + void BeginSendPhase() + { + PCMS_FUNCTION_TIMER; + channel_.BeginSendCommunicationPhase(); + } + void EndSendPhase() + { + PCMS_FUNCTION_TIMER; + channel_.EndSendCommunicationPhase(); + } + void BeginReceivePhase() + { + PCMS_FUNCTION_TIMER; + channel_.BeginReceiveCommunicationPhase(); + } + void EndReceivePhase() + { + PCMS_FUNCTION_TIMER; + channel_.EndReceiveCommunicationPhase(); + } + +private: + std::string name_; + MPI_Comm mpi_comm_; + redev::Redev redev_; + // map rather than unordered_map is necessary to avoid iterator invalidation. + // This is important because we pass pointers to the fields out of this class + std::map fields_; + redev::Channel channel_; +}; +} // namespace pcms + +#endif // PCMS_COUPLER_H From fc32002a4b971a9a1ebce59e5bee830e0a2099df Mon Sep 17 00:00:00 2001 From: Angelyr Date: Fri, 28 Mar 2025 13:24:43 -0400 Subject: [PATCH 02/12] removed client.h --- src/CMakeLists.txt | 4 - src/pcms.h | 3 - src/pcms/capi/client.cpp | 23 ++--- src/pcms/client.h | 113 ------------------------ src/pcms/coupler.h | 9 +- test/test_proxy_coupling.cpp | 6 +- test/test_proxy_coupling_xgc_server.cpp | 1 - test/xgc_n0_coupling_server.cpp | 1 - 8 files changed, 20 insertions(+), 140 deletions(-) delete mode 100644 src/pcms/client.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b9d240ef..35bb7ad9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -68,10 +68,6 @@ endif() if(PCMS_ENABLE_SERVER) target_compile_definitions(pcms_core PUBLIC -DPCMS_HAS_SERVER) endif() -if(PCMS_ENABLE_CLIENT) - list(APPEND PCMS_HEADERS pcms/client.h) - target_compile_definitions(pcms_core PUBLIC -DPCMS_HAS_CLIENT) -endif() if(PCMS_HAS_ASAN) target_compile_options(pcms_core PRIVATE -fsanitize=address -fno-omit-frame-pointer) diff --git a/src/pcms.h b/src/pcms.h index 3fed6ca7..14eb2241 100644 --- a/src/pcms.h +++ b/src/pcms.h @@ -4,8 +4,5 @@ #ifdef PCMS_HAS_SERVER #include "pcms/server.h" #endif -#ifdef PCMS_HAS_CLIENT -#include "pcms/client.h" -#endif #endif diff --git a/src/pcms/capi/client.cpp b/src/pcms/capi/client.cpp index 03017b8d..1909871f 100644 --- a/src/pcms/capi/client.cpp +++ b/src/pcms/capi/client.cpp @@ -1,6 +1,7 @@ #include "client.h" #include "pcms.h" #include "pcms/xgc_field_adapter.h" +#include "pcms/coupler.h" #include #include // #ifdef PCMS_HAS_OMEGA_H @@ -29,13 +30,13 @@ using FieldAdapterVariant = [[nodiscard]] PcmsClientHandle pcms_create_client(const char* name, MPI_Comm comm) { - auto* client = new pcms::CouplerClient(name, comm); + auto* client = new pcms::Coupler(name, comm); return {reinterpret_cast(client)}; } void pcms_destroy_client(PcmsClientHandle client) { if (client.pointer != nullptr) - delete reinterpret_cast(client.pointer); + delete reinterpret_cast(client.pointer); } PcmsReverseClassificationHandle pcms_load_reverse_classification( const char* file, MPI_Comm comm) @@ -52,7 +53,7 @@ void pcms_destroy_reverse_classification(PcmsReverseClassificationHandle rc) } struct AddFieldVariantOperators { - AddFieldVariantOperators(const char* name, pcms::CouplerClient* client, + AddFieldVariantOperators(const char* name, pcms::Coupler* client, int participates) : name_(name), client_(client), participates_(participates) { @@ -72,7 +73,7 @@ struct AddFieldVariantOperators } const char* name_; - pcms::CouplerClient* client_; + pcms::Coupler* client_; bool participates_; }; @@ -83,7 +84,7 @@ PcmsFieldHandle pcms_add_field(PcmsClientHandle client_handle, const char* name, auto* adapter = reinterpret_cast(adapter_handle.pointer); - auto* client = reinterpret_cast(client_handle.pointer); + auto* client = reinterpret_cast(client_handle.pointer); PCMS_ALWAYS_ASSERT(client != nullptr); PCMS_ALWAYS_ASSERT(adapter != nullptr); // pcms::CoupledField* field = std::visit( @@ -99,13 +100,13 @@ PcmsFieldHandle pcms_add_field(PcmsClientHandle client_handle, const char* name, } void pcms_send_field_name(PcmsClientHandle client_handle, const char* name) { - auto* client = reinterpret_cast(client_handle.pointer); + auto* client = reinterpret_cast(client_handle.pointer); PCMS_ALWAYS_ASSERT(client != nullptr); client->SendField(name); } void pcms_receive_field_name(PcmsClientHandle client_handle, const char* name) { - auto* client = reinterpret_cast(client_handle.pointer); + auto* client = reinterpret_cast(client_handle.pointer); PCMS_ALWAYS_ASSERT(client != nullptr); client->ReceiveField(name); } @@ -198,25 +199,25 @@ int pcms_reverse_classification_count_verts(PcmsReverseClassificationHandle rc) } void pcms_begin_send_phase(PcmsClientHandle h) { - auto* client = reinterpret_cast(h.pointer); + auto* client = reinterpret_cast(h.pointer); PCMS_ALWAYS_ASSERT(client != nullptr); client->BeginSendPhase(); } void pcms_end_send_phase(PcmsClientHandle h) { - auto* client = reinterpret_cast(h.pointer); + auto* client = reinterpret_cast(h.pointer); PCMS_ALWAYS_ASSERT(client != nullptr); client->EndSendPhase(); } void pcms_begin_receive_phase(PcmsClientHandle h) { - auto* client = reinterpret_cast(h.pointer); + auto* client = reinterpret_cast(h.pointer); PCMS_ALWAYS_ASSERT(client != nullptr); client->BeginReceivePhase(); } void pcms_end_receive_phase(PcmsClientHandle h) { - auto* client = reinterpret_cast(h.pointer); + auto* client = reinterpret_cast(h.pointer); PCMS_ALWAYS_ASSERT(client != nullptr); client->EndReceivePhase(); } \ No newline at end of file diff --git a/src/pcms/client.h b/src/pcms/client.h deleted file mode 100644 index 1aa95c22..00000000 --- a/src/pcms/client.h +++ /dev/null @@ -1,113 +0,0 @@ -#ifndef PCMS_COUPLING_CLIENT_H -#define PCMS_COUPLING_CLIENT_H -#include "pcms/common.h" -#include "pcms/field_communicator.h" -#include "pcms/profile.h" -#include "pcms/coupler.h" - -namespace pcms -{ - -class CouplerClient -{ -public: - CouplerClient(std::string name, MPI_Comm comm, - redev::TransportType transport_type = redev::TransportType::BP4, - adios2::Params params = {{"Streaming", "On"}, - {"OpenTimeoutSecs", "400"}}, - std::string path = "") - : name_(std::move(name)), - mpi_comm_(comm), - redev_(comm), - channel_{redev_.CreateAdiosChannel(name_, std::move(params), - transport_type, std::move(path))} - { - PCMS_FUNCTION_TIMER; - } - - [[nodiscard]] const redev::Partition& GetPartition() const - { - PCMS_FUNCTION_TIMER; - return redev_.GetPartition(); - } - - /** - * @known_issue - * The redev partion has to be same as the partition of the OH mesh. - * It asserts the number of elements sent and received are same. - * otherwise, ConstructPermutation() will fail. - */ - template - CoupledField* AddField(std::string name, FieldAdapterT field_adapter, - bool participates = true) - { - PCMS_FUNCTION_TIMER; - auto [it, inserted] = - fields_.template try_emplace(name, name, std::move(field_adapter), - mpi_comm_, redev_, channel_, participates); - if (!inserted) { - std::cerr << "OHField with this name" << name << "already exists!\n"; - std::terminate(); - } - return &(it->second); - } - - // take a string& since map cannot be searched with string_view - // (heterogeneous lookup) - void SendField(const std::string& name, Mode mode = Mode::Synchronous) - { - PCMS_FUNCTION_TIMER; - PCMS_ALWAYS_ASSERT(InSendPhase()); - detail::find_or_error(name, fields_).Send(mode); - }; - // take a string& since map cannot be searched with string_view - // (heterogeneous lookup) - void ReceiveField(const std::string& name) - { - PCMS_FUNCTION_TIMER; - PCMS_ALWAYS_ASSERT(InReceivePhase()); - detail::find_or_error(name, fields_).Receive(); - }; - [[nodiscard]] bool InSendPhase() const noexcept - { - PCMS_FUNCTION_TIMER; - return channel_.InSendCommunicationPhase(); - } - [[nodiscard]] bool InReceivePhase() const noexcept - { - PCMS_FUNCTION_TIMER; - return channel_.InReceiveCommunicationPhase(); - } - void BeginSendPhase() - { - PCMS_FUNCTION_TIMER; - channel_.BeginSendCommunicationPhase(); - } - void EndSendPhase() - { - PCMS_FUNCTION_TIMER; - channel_.EndSendCommunicationPhase(); - } - void BeginReceivePhase() - { - PCMS_FUNCTION_TIMER; - channel_.BeginReceiveCommunicationPhase(); - } - void EndReceivePhase() - { - PCMS_FUNCTION_TIMER; - channel_.EndReceiveCommunicationPhase(); - } - -private: - std::string name_; - MPI_Comm mpi_comm_; - redev::Redev redev_; - // map rather than unordered_map is necessary to avoid iterator invalidation. - // This is important because we pass pointers to the fields out of this class - std::map fields_; - redev::Channel channel_; -}; -} // namespace pcms - -#endif // PCMS_COUPLING_CLIENT_H diff --git a/src/pcms/coupler.h b/src/pcms/coupler.h index a57f4ab9..a5d61074 100644 --- a/src/pcms/coupler.h +++ b/src/pcms/coupler.h @@ -91,21 +91,22 @@ class CoupledField class Coupler { private: - redev::Redev SetUpRedev(bool isRdv, redev::Partition partition) { - if (isRdv) + redev::Redev SetUpRedev(bool isServer, redev::Partition partition) { + if (isServer) return redev::Redev(mpi_comm_, std::move(partition)); else return redev::Redev(mpi_comm_); } public: - Coupler(std::string name, MPI_Comm comm, bool isRdv, redev::Partition partition, + Coupler(std::string name, MPI_Comm comm, bool isServer = false, + redev::Partition partition = redev::Partition{redev::RCBPtn()}, redev::TransportType transport_type = redev::TransportType::BP4, adios2::Params params = {{"Streaming", "On"}, {"OpenTimeoutSecs", "60"}}, std::string path = "") : name_(std::move(name)), mpi_comm_(comm), - redev_(SetUpRedev(isRdv, std::move(partition))), + redev_(SetUpRedev(isServer, std::move(partition))), channel_{redev_.CreateAdiosChannel(name_, std::move(params), transport_type, std::move(path))} { diff --git a/test/test_proxy_coupling.cpp b/test/test_proxy_coupling.cpp index 43133341..4a19c77f 100644 --- a/test/test_proxy_coupling.cpp +++ b/test/test_proxy_coupling.cpp @@ -11,7 +11,7 @@ #include using pcms::Copy; -using pcms::CouplerClient; +using pcms::Coupler; using pcms::CouplerServer; using pcms::GO; using pcms::Lagrange; @@ -27,7 +27,7 @@ namespace ts = test_support; void xgc_delta_f(MPI_Comm comm, Omega_h::Mesh& mesh) { - CouplerClient cpl("proxy_couple_xgc_delta_f", comm); + Coupler cpl("proxy_couple_xgc_delta_f", comm); auto is_overlap = ts::markOverlapMeshEntities(mesh, ts::IsModelEntInOverlap{}); cpl.AddField("gids", @@ -49,7 +49,7 @@ void xgc_delta_f(MPI_Comm comm, Omega_h::Mesh& mesh) } void xgc_total_f(MPI_Comm comm, Omega_h::Mesh& mesh) { - pcms::CouplerClient cpl("proxy_couple_xgc_total_f", comm); + pcms::Coupler cpl("proxy_couple_xgc_total_f", comm); auto is_overlap = ts::markOverlapMeshEntities(mesh, ts::IsModelEntInOverlap{}); cpl.AddField("gids", OmegaHFieldAdapter("global", mesh, is_overlap)); diff --git a/test/test_proxy_coupling_xgc_server.cpp b/test/test_proxy_coupling_xgc_server.cpp index 086652f0..b2526564 100644 --- a/test/test_proxy_coupling_xgc_server.cpp +++ b/test/test_proxy_coupling_xgc_server.cpp @@ -9,7 +9,6 @@ using pcms::ConstructRCFromOmegaHMesh; using pcms::Copy; -using pcms::CouplerClient; using pcms::CouplerServer; using pcms::GO; using pcms::Lagrange; diff --git a/test/xgc_n0_coupling_server.cpp b/test/xgc_n0_coupling_server.cpp index 68490f41..7dc40740 100644 --- a/test/xgc_n0_coupling_server.cpp +++ b/test/xgc_n0_coupling_server.cpp @@ -8,7 +8,6 @@ #include using pcms::Copy; -using pcms::CouplerClient; using pcms::CouplerServer; using pcms::GO; using pcms::LO; From 31efae8db11561ed3c328b494b5561f7a9b8a344 Mon Sep 17 00:00:00 2001 From: Angelyr Date: Fri, 28 Mar 2025 12:34:00 -0700 Subject: [PATCH 03/12] fixed bug --- test/test_proxy_coupling.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_proxy_coupling.cpp b/test/test_proxy_coupling.cpp index 4a19c77f..2a60714b 100644 --- a/test/test_proxy_coupling.cpp +++ b/test/test_proxy_coupling.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -11,7 +12,6 @@ #include using pcms::Copy; -using pcms::Coupler; using pcms::CouplerServer; using pcms::GO; using pcms::Lagrange; @@ -27,7 +27,7 @@ namespace ts = test_support; void xgc_delta_f(MPI_Comm comm, Omega_h::Mesh& mesh) { - Coupler cpl("proxy_couple_xgc_delta_f", comm); + pcms::Coupler cpl("proxy_couple_xgc_delta_f", comm); auto is_overlap = ts::markOverlapMeshEntities(mesh, ts::IsModelEntInOverlap{}); cpl.AddField("gids", From cb8d48a2d8ba51b9a67dbb87b1b60930851fc77c Mon Sep 17 00:00:00 2001 From: Angelyr Date: Fri, 28 Mar 2025 13:26:22 -0700 Subject: [PATCH 04/12] similar coupled fields --- src/pcms/coupler.h | 36 ++++++++++++++++++++++++++++++------ src/pcms/server.h | 37 ++----------------------------------- 2 files changed, 32 insertions(+), 41 deletions(-) diff --git a/src/pcms/coupler.h b/src/pcms/coupler.h index a5d61074..e66757a4 100644 --- a/src/pcms/coupler.h +++ b/src/pcms/coupler.h @@ -36,15 +36,29 @@ class CoupledField PCMS_FUNCTION_TIMER; coupled_field_->Send(mode); } - void Receive() + void Receive(Mode mode = Mode::Synchronous) { PCMS_FUNCTION_TIMER; - coupled_field_->Receive(); + coupled_field_->Receive(mode); + } + template + [[nodiscard]] T* GetFieldAdapter() const + { + PCMS_FUNCTION_TIMER; + if (typeid(T) == coupled_field_->GetFieldAdapterType()) { + auto* adapter = coupled_field_->GetFieldAdapter(); + return reinterpret_cast(adapter); + } + std::cerr << "Requested type does not match field adapter type\n"; + std::abort(); } struct CoupledFieldConcept { virtual void Send(Mode) = 0; - virtual void Receive() = 0; + virtual void Receive(Mode) = 0; + [[nodiscard]] virtual const std::type_info& GetFieldAdapterType() + const noexcept = 0; + [[nodiscard]] virtual void* GetFieldAdapter() noexcept = 0; virtual ~CoupledFieldConcept() = default; }; template @@ -58,7 +72,8 @@ class CoupledField : mpi_comm_subset_(mpi_comm_subset), field_adapter_(std::move(field_adapter)), comm_(FieldCommunicator(name, mpi_comm_subset_, redev, channel, - field_adapter_)) + field_adapter_)), + type_info_(typeid(FieldAdapterT)) { PCMS_FUNCTION_TIMER; } @@ -67,10 +82,18 @@ class CoupledField PCMS_FUNCTION_TIMER; comm_.Send(mode); }; - void Receive() final + void Receive(Mode mode) final { PCMS_FUNCTION_TIMER; - comm_.Receive(); + comm_.Receive(mode); + }; + virtual const std::type_info& GetFieldAdapterType() const noexcept + { + return type_info_; + } + virtual void* GetFieldAdapter() noexcept + { + return reinterpret_cast(&field_adapter_); }; ~CoupledFieldModel() { @@ -82,6 +105,7 @@ class CoupledField MPI_Comm mpi_comm_subset_; FieldAdapterT field_adapter_; FieldCommunicator comm_; + const std::type_info& type_info_; }; private: diff --git a/src/pcms/server.h b/src/pcms/server.h index eb74d50e..115044ed 100644 --- a/src/pcms/server.h +++ b/src/pcms/server.h @@ -14,27 +14,10 @@ namespace pcms class ConvertibleCoupledField { public: - template - ConvertibleCoupledField(const std::string& name, FieldAdapterT field_adapter, - FieldCommunicator field_comm, - Omega_h::Mesh& internal_mesh, - Omega_h::Read internal_field_mask = {}) - : internal_field_{OmegaHField( - name + ".__internal__", internal_mesh, internal_field_mask, "", 10, 10, field_adapter.GetEntityType())} - { - PCMS_FUNCTION_TIMER; - coupled_field_ = std::make_unique>( - std::move(field_adapter), std::move(field_comm)); - } template ConvertibleCoupledField(const std::string& name, FieldAdapterT field_adapter, MPI_Comm mpi_comm, redev::Redev& redev, - redev::Channel& channel, Omega_h::Mesh& internal_mesh, - Omega_h::Read internal_field_mask) - : internal_field_{OmegaHField( - name + ".__internal__", internal_mesh, internal_field_mask, "", 10, 10, field_adapter.GetEntityType())} + redev::Channel& channel) { PCMS_FUNCTION_TIMER; coupled_field_ = @@ -52,16 +35,6 @@ class ConvertibleCoupledField PCMS_FUNCTION_TIMER; coupled_field_->Receive(mode); } - [[nodiscard]] InternalField& GetInternalField() noexcept - { - PCMS_FUNCTION_TIMER; - return internal_field_; - } - [[nodiscard]] const InternalField& GetInternalField() const noexcept - { - PCMS_FUNCTION_TIMER; - return internal_field_; - } template [[nodiscard]] T* GetFieldAdapter() const { @@ -131,11 +104,6 @@ class ConvertibleCoupledField private: std::unique_ptr coupled_field_; - // even though we know the type of the internal field, - // we store it as the InternalField variant since this avoids any copies - // This comes at the cost of a slightly larger type with need to use the get<> - // function - InternalField internal_field_; }; // TODO: strategy to merge Server/CLient Application and Fields class Application @@ -163,8 +131,7 @@ class Application PCMS_FUNCTION_TIMER; auto [it, inserted] = fields_.template try_emplace( name, name, std::forward(field_adapter), mpi_comm_, redev_, - channel_, internal_mesh_, - internal_field_mask); + channel_); if (!inserted) { std::cerr << "OHField with this name" << name << "already exists!\n"; std::terminate(); From f7d5eb2f6298ac8b59fa41e9be845d6f357ba177 Mon Sep 17 00:00:00 2001 From: Angelyr Date: Fri, 28 Mar 2025 14:03:41 -0700 Subject: [PATCH 05/12] removed usages of omega_h --- src/pcms/coupler.h | 17 +++++++++-- src/pcms/server.h | 38 ++++++------------------- test/test_proxy_coupling.cpp | 11 +++---- test/test_proxy_coupling_xgc_server.cpp | 8 +++--- test/xgc_n0_coupling_server.cpp | 5 ++-- 5 files changed, 33 insertions(+), 46 deletions(-) diff --git a/src/pcms/coupler.h b/src/pcms/coupler.h index e66757a4..9e70ab73 100644 --- a/src/pcms/coupler.h +++ b/src/pcms/coupler.h @@ -174,11 +174,11 @@ class Coupler }; // take a string& since map cannot be searched with string_view // (heterogeneous lookup) - void ReceiveField(const std::string& name) + void ReceiveField(const std::string& name, Mode mode = Mode::Synchronous) { PCMS_FUNCTION_TIMER; PCMS_ALWAYS_ASSERT(InReceivePhase()); - detail::find_or_error(name, fields_).Receive(); + detail::find_or_error(name, fields_).Receive(mode); }; [[nodiscard]] bool InSendPhase() const noexcept { @@ -211,6 +211,19 @@ class Coupler channel_.EndReceiveCommunicationPhase(); } + template + auto SendPhase(const Func& func, Args&&... args) + { + PCMS_FUNCTION_TIMER; + return channel_.SendPhase(func, std::forward(args)...); + } + template + auto ReceivePhase(const Func& func, Args&&... args) + { + PCMS_FUNCTION_TIMER; + return channel_.ReceivePhase(func, std::forward(args)...); + } + private: std::string name_; MPI_Comm mpi_comm_; diff --git a/src/pcms/server.h b/src/pcms/server.h index 115044ed..52baf04d 100644 --- a/src/pcms/server.h +++ b/src/pcms/server.h @@ -110,14 +110,13 @@ class Application { public: Application(std::string name, redev::Redev& rdv, MPI_Comm comm, - redev::Redev& redev, Omega_h::Mesh& internal_mesh, - adios2::Params params, redev::TransportType transport_type, + redev::Redev& redev, adios2::Params params, + redev::TransportType transport_type, std::string path) : mpi_comm_(comm), redev_(redev), channel_{rdv.CreateAdiosChannel(std::move(name), std::move(params), - transport_type, std::move(path))}, - internal_mesh_{internal_mesh} + transport_type, std::move(path))} { PCMS_FUNCTION_TIMER; } @@ -125,15 +124,14 @@ class Application // These fields are supposed to be agnostic to adios2... template ConvertibleCoupledField* AddField( - std::string name, FieldAdapterT&& field_adapter, - Omega_h::Read internal_field_mask = {}) + std::string name, FieldAdapterT&& field_adapter) { PCMS_FUNCTION_TIMER; auto [it, inserted] = fields_.template try_emplace( name, name, std::forward(field_adapter), mpi_comm_, redev_, channel_); if (!inserted) { - std::cerr << "OHField with this name" << name << "already exists!\n"; + std::cerr << "Field with this name" << name << "already exists!\n"; std::terminate(); } return &(it->second); @@ -202,18 +200,15 @@ class Application // internal data and rehash of unordered_map can cause pointer invalidation. // map is less cache friendly, but pointers are not invalidated. std::map fields_; - Omega_h::Mesh& internal_mesh_; }; class CouplerServer { public: - CouplerServer(std::string name, MPI_Comm comm, redev::Partition partition, - Omega_h::Mesh& mesh) + CouplerServer(std::string name, MPI_Comm comm, redev::Partition partition) : name_(std::move(name)), mpi_comm_(comm), - redev_({comm, std::move(partition), ProcessType::Server}), - internal_mesh_(mesh) + redev_({comm, std::move(partition), ProcessType::Server}) { PCMS_FUNCTION_TIMER; } @@ -225,7 +220,7 @@ class CouplerServer PCMS_FUNCTION_TIMER; auto key = path + name; auto [it, inserted] = applications_.template try_emplace( - key, std::move(name), redev_, mpi_comm_, redev_, internal_mesh_, + key, std::move(name), redev_, mpi_comm_, redev_, std::move(params), transport_type, std::move(path)); if (!inserted) { std::cerr << "Application with name " << name << "already exists!\n"; @@ -238,30 +233,13 @@ class CouplerServer { return redev_.GetPartition(); } - [[nodiscard]] Omega_h::Mesh& GetMesh() noexcept { return internal_mesh_; } - - [[nodiscard]] const auto& GetInternalFields() const noexcept - { - return internal_fields_; - } - - // TODO: consider an "advanced api" wrapper of some sort and protect direct - // access to the internal fields with passkey idom or some other way. User - // could get unexpected behavior if they mess with the internal field map. - /// This function should not be used directly it is experimental for Philip - /// to do Benesh development. Expect that it will be removed in the future. - [[nodiscard]] auto& GetInternalFields() noexcept { return internal_fields_; } private: std::string name_; MPI_Comm mpi_comm_; redev::Redev redev_; - // xgc_coupler owns internal fields since both gather/scatter ops use these - // these internal fields correspond to the "Combined" fields - std::map internal_fields_; // gather and scatter operations have reference to internal fields std::map applications_; - Omega_h::Mesh& internal_mesh_; }; } // namespace pcms #endif // PCMS_COUPLING_SERVER_H diff --git a/test/test_proxy_coupling.cpp b/test/test_proxy_coupling.cpp index 2a60714b..8fc9f173 100644 --- a/test/test_proxy_coupling.cpp +++ b/test/test_proxy_coupling.cpp @@ -71,7 +71,7 @@ void xgc_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, std::string_view cpn_file) // user responsibility to keep it alive! pcms::CouplerServer cpl( "proxy_couple", comm, - redev::Partition{ts::setupServerPartition(mesh, cpn_file)}, mesh); + redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); const auto partition = std::get(cpl.GetPartition()); auto is_overlap = ts::markServerOverlapRegion(mesh, partition, ts::IsModelEntInOverlap{}); @@ -79,14 +79,11 @@ void xgc_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, std::string_view cpn_file) auto* delta_f = cpl.AddApplication("proxy_couple_xgc_delta_f"); // TODO, fields should have a transfer policy rather than parameters auto* total_f_gids = total_f->AddField( - "gids", OmegaHFieldAdapter("total_f_gids", mesh, is_overlap), - is_overlap); + "gids", OmegaHFieldAdapter("total_f_gids", mesh, is_overlap)); auto* delta_f_gids = delta_f->AddField( - "gids", OmegaHFieldAdapter("delta_f_gids", mesh, is_overlap), - is_overlap); + "gids", OmegaHFieldAdapter("delta_f_gids", mesh, is_overlap)); auto* delta_f_gids2 = delta_f->AddField( - "gids2", OmegaHFieldAdapter("delta_f_gids2", mesh, is_overlap), - is_overlap); + "gids2", OmegaHFieldAdapter("delta_f_gids2", mesh, is_overlap)); do { for (int i = 0; i < COMM_ROUNDS; ++i) { total_f->ReceivePhase([&]() { total_f_gids->Receive(); }); diff --git a/test/test_proxy_coupling_xgc_server.cpp b/test/test_proxy_coupling_xgc_server.cpp index b2526564..b62be254 100644 --- a/test/test_proxy_coupling_xgc_server.cpp +++ b/test/test_proxy_coupling_xgc_server.cpp @@ -28,7 +28,7 @@ void xgc_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, std::string_view cpn_file) // user responsibility to keep it alive! pcms::CouplerServer cpl( "proxy_couple_server", comm, - redev::Partition{ts::setupServerPartition(mesh, cpn_file)}, mesh); + redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); const auto partition = std::get(cpl.GetPartition()); ReverseClassificationVertex rc; if (mesh.has_tag(0, "simNumbering")) { @@ -51,7 +51,7 @@ void xgc_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, std::string_view cpn_file) auto field_adapter = pcms::XGCFieldAdapter( ss.str(), comm, make_array_view(data[i]), rc, ts::IsModelEntInOverlap{}); fields.push_back( - application->AddField(ss.str(), std::move(field_adapter), is_overlap)); + application->AddField(ss.str(), std::move(field_adapter))); } do { @@ -83,7 +83,7 @@ void omegah_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, // user responsibility to keep it alive! pcms::CouplerServer cpl( "proxy_couple_server", comm, - redev::Partition{ts::setupServerPartition(mesh, cpn_file)}, mesh); + redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); const auto partition = std::get(cpl.GetPartition()); auto* application = cpl.AddApplication("proxy_couple"); std::string numbering; @@ -108,7 +108,7 @@ void omegah_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, auto field_adapter = pcms::OmegaHFieldAdapter(ss.str(), mesh, is_overlap, numbering); fields.push_back( - application->AddField(ss.str(), std::move(field_adapter), is_overlap)); + application->AddField(ss.str(), std::move(field_adapter))); } do { application->ReceivePhase([&]() { diff --git a/test/xgc_n0_coupling_server.cpp b/test/xgc_n0_coupling_server.cpp index 7dc40740..578b4877 100644 --- a/test/xgc_n0_coupling_server.cpp +++ b/test/xgc_n0_coupling_server.cpp @@ -30,8 +30,7 @@ static pcms::ConvertibleCoupledField* AddField(pcms::Application *application, c } return application->AddField(field_name.str(), pcms::OmegaHFieldAdapter( - path+field_name.str(), mesh, is_overlap, numbering), - is_overlap); + path+field_name.str(), mesh, is_overlap, numbering)); } struct XGCAnalysis { @@ -192,7 +191,7 @@ void omegah_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, pcms::CouplerServer cpl("xgc_n0_coupling", comm, - redev::Partition{ts::setupServerPartition(mesh, cpn_file)}, mesh); + redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); const auto partition = std::get(cpl.GetPartition()); std::string numbering = "simNumbering"; PCMS_ALWAYS_ASSERT(mesh.has_tag(0, numbering)); From 5c147b018b2d219b8c3ad5529ccb678b5af064be Mon Sep 17 00:00:00 2001 From: Angelyr Date: Fri, 28 Mar 2025 14:10:31 -0700 Subject: [PATCH 06/12] removed convertible coupled field --- src/pcms/coupler.h | 2 +- src/pcms/server.h | 101 +----------------------- test/field_transfer_example.cpp | 2 +- test/test_proxy_coupling_xgc_server.cpp | 20 ++--- test/xgc_n0_coupling_server.cpp | 20 ++--- 5 files changed, 25 insertions(+), 120 deletions(-) diff --git a/src/pcms/coupler.h b/src/pcms/coupler.h index 9e70ab73..c52fc00e 100644 --- a/src/pcms/coupler.h +++ b/src/pcms/coupler.h @@ -13,7 +13,7 @@ class CoupledField template CoupledField(const std::string& name, FieldAdapterT field_adapter, MPI_Comm mpi_comm, redev::Redev& redev, redev::Channel& channel, - bool participates) + bool participates = true) { PCMS_FUNCTION_TIMER; MPI_Comm mpi_comm_subset = MPI_COMM_NULL; diff --git a/src/pcms/server.h b/src/pcms/server.h index 52baf04d..032c2857 100644 --- a/src/pcms/server.h +++ b/src/pcms/server.h @@ -4,107 +4,12 @@ #include "pcms/field_communicator.h" #include "pcms/omega_h_field.h" #include "pcms/profile.h" +#include "pcms/coupler.h" #include #include namespace pcms { -// TODO: come up with better name for this...Don't like CoupledFieldServer -// because it's necessarily tied to the Server of the xgc_coupler -class ConvertibleCoupledField -{ -public: - template - ConvertibleCoupledField(const std::string& name, FieldAdapterT field_adapter, - MPI_Comm mpi_comm, redev::Redev& redev, - redev::Channel& channel) - { - PCMS_FUNCTION_TIMER; - coupled_field_ = - std::make_unique>( - name, std::move(field_adapter), mpi_comm, redev, channel); - } - - void Send(Mode mode = Mode::Synchronous) - { - PCMS_FUNCTION_TIMER; - coupled_field_->Send(mode); - } - void Receive(Mode mode=Mode::Synchronous) - { - PCMS_FUNCTION_TIMER; - coupled_field_->Receive(mode); - } - template - [[nodiscard]] T* GetFieldAdapter() const - { - PCMS_FUNCTION_TIMER; - if (typeid(T) == coupled_field_->GetFieldAdapterType()) { - auto* adapter = coupled_field_->GetFieldAdapter(); - return reinterpret_cast(adapter); - } - std::cerr << "Requested type does not match field adapter type\n"; - std::abort(); - } - struct CoupledFieldConcept - { - virtual void Send(Mode) = 0; - virtual void Receive(Mode) = 0; - [[nodiscard]] virtual const std::type_info& GetFieldAdapterType() - const noexcept = 0; - [[nodiscard]] virtual void* GetFieldAdapter() noexcept = 0; - virtual ~CoupledFieldConcept() = default; - }; - template - struct CoupledFieldModel final : CoupledFieldConcept - { - using value_type = typename FieldAdapterT::value_type; - - CoupledFieldModel(FieldAdapterT&& field_adapter, - FieldCommunicator&& comm) - : field_adapter_(std::move(field_adapter)), - comm_(std::move(comm)), - type_info_(typeid(FieldAdapterT)) - { - PCMS_FUNCTION_TIMER; - } - CoupledFieldModel(const std::string& name, FieldAdapterT&& field_adapter, - MPI_Comm mpi_comm, redev::Redev& redev, - redev::Channel& channel) - : field_adapter_(std::move(field_adapter)), - comm_(FieldCommunicator(name, mpi_comm, redev, channel, - field_adapter_)), - type_info_(typeid(FieldAdapterT)) - { - PCMS_FUNCTION_TIMER; - } - void Send(Mode mode) final - { - PCMS_FUNCTION_TIMER; - comm_.Send(mode); - }; - void Receive(Mode mode) final - { - PCMS_FUNCTION_TIMER; - comm_.Receive(mode); - }; - virtual const std::type_info& GetFieldAdapterType() const noexcept - { - return type_info_; - } - virtual void* GetFieldAdapter() noexcept - { - return reinterpret_cast(&field_adapter_); - }; - - FieldAdapterT field_adapter_; - FieldCommunicator comm_; - const std::type_info& type_info_; - }; - -private: - std::unique_ptr coupled_field_; -}; // TODO: strategy to merge Server/CLient Application and Fields class Application { @@ -123,7 +28,7 @@ class Application // FIXME should take a file path for the parameters, not take adios2 params. // These fields are supposed to be agnostic to adios2... template - ConvertibleCoupledField* AddField( + CoupledField* AddField( std::string name, FieldAdapterT&& field_adapter) { PCMS_FUNCTION_TIMER; @@ -199,7 +104,7 @@ class Application // map is used rather than unordered_map because we give pointers to the // internal data and rehash of unordered_map can cause pointer invalidation. // map is less cache friendly, but pointers are not invalidated. - std::map fields_; + std::map fields_; }; class CouplerServer diff --git a/test/field_transfer_example.cpp b/test/field_transfer_example.cpp index c0fade6c..8664b5cb 100644 --- a/test/field_transfer_example.cpp +++ b/test/field_transfer_example.cpp @@ -41,7 +41,7 @@ struct MeanCombiner void SetApplicationFields(const OHField& app_a_field, const OHField& app_b_field); -using pcms::ConvertibleCoupledField; +using pcms::CoupledField; using pcms::ProcessType; using pcms::FieldCommunicator; using pcms::InternalField; diff --git a/test/test_proxy_coupling_xgc_server.cpp b/test/test_proxy_coupling_xgc_server.cpp index b62be254..d3af7fc6 100644 --- a/test/test_proxy_coupling_xgc_server.cpp +++ b/test/test_proxy_coupling_xgc_server.cpp @@ -43,7 +43,7 @@ void xgc_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, std::string_view cpn_file) constexpr int nplanes = 2; std::array, nplanes> data; - std::vector fields; + std::vector fields; for (int i = 0; i < nplanes; ++i) { data[i].resize(mesh.nverts()); std::stringstream ss; @@ -57,19 +57,19 @@ void xgc_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, std::string_view cpn_file) do { application->ReceivePhase([&]() { std::for_each(fields.begin(), fields.end(), - [](pcms::ConvertibleCoupledField* f) { f->Receive(); }); + [](pcms::CoupledField* f) { f->Receive(); }); }); application->SendPhase([&]() { std::for_each(fields.begin(), fields.end(), - [](pcms::ConvertibleCoupledField* f) { f->Send(); }); + [](pcms::CoupledField* f) { f->Send(); }); }); application->ReceivePhase([&]() { std::for_each(fields.begin(), fields.end(), - [](pcms::ConvertibleCoupledField* f) { f->Receive(); }); + [](pcms::CoupledField* f) { f->Receive(); }); }); application->SendPhase([&]() { std::for_each(fields.begin(), fields.end(), - [](pcms::ConvertibleCoupledField* f) { f->Send(); }); + [](pcms::CoupledField* f) { f->Send(); }); }); } while (!done); @@ -101,7 +101,7 @@ void omegah_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, auto is_overlap = ts::markServerOverlapRegion(mesh, partition, ts::IsModelEntInOverlap{}); constexpr int nplanes = 2; - std::vector fields; + std::vector fields; for (int i = 0; i < nplanes; ++i) { std::stringstream ss; ss << "xgc_gids_plane_" << i; @@ -113,19 +113,19 @@ void omegah_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, do { application->ReceivePhase([&]() { std::for_each(fields.begin(), fields.end(), - [](pcms::ConvertibleCoupledField* f) { f->Receive(); }); + [](pcms::CoupledField* f) { f->Receive(); }); }); application->SendPhase([&]() { std::for_each(fields.begin(), fields.end(), - [](pcms::ConvertibleCoupledField* f) { f->Send(); }); + [](pcms::CoupledField* f) { f->Send(); }); }); application->ReceivePhase([&]() { std::for_each(fields.begin(), fields.end(), - [](pcms::ConvertibleCoupledField* f) { f->Receive(); }); + [](pcms::CoupledField* f) { f->Receive(); }); }); application->SendPhase([&]() { std::for_each(fields.begin(), fields.end(), - [](pcms::ConvertibleCoupledField* f) { f->Send(); }); + [](pcms::CoupledField* f) { f->Send(); }); }); } while (!done); Omega_h::vtk::write_parallel("proxy_couple", &mesh, mesh.dim()); diff --git a/test/xgc_n0_coupling_server.cpp b/test/xgc_n0_coupling_server.cpp index 578b4877..250da3be 100644 --- a/test/xgc_n0_coupling_server.cpp +++ b/test/xgc_n0_coupling_server.cpp @@ -21,7 +21,7 @@ namespace ts = test_support; // [[nodiscard]] -static pcms::ConvertibleCoupledField* AddField(pcms::Application *application, const std::string& name, const std::string& path, Omega_h::Read is_overlap, const std::string& numbering, Omega_h::Mesh& mesh, int plane) { +static pcms::CoupledField* AddField(pcms::Application *application, const std::string& name, const std::string& path, Omega_h::Read is_overlap, const std::string& numbering, Omega_h::Mesh& mesh, int plane) { PCMS_ALWAYS_ASSERT(application != nullptr); std::stringstream field_name; field_name << name; @@ -34,27 +34,27 @@ static pcms::ConvertibleCoupledField* AddField(pcms::Application *application, c } struct XGCAnalysis { - using FieldVec = std::vector; + using FieldVec = std::vector; std::array dpot; FieldVec pot0; std::array edensity; std::array idensity; - pcms::ConvertibleCoupledField* psi; - pcms::ConvertibleCoupledField* gids; + pcms::CoupledField* psi; + pcms::CoupledField* gids; }; -static void ReceiveFields(const std::vector & fields) { +static void ReceiveFields(const std::vector & fields) { for(auto* field : fields) { field->Receive(); } } -static void SendFields(const std::vector & fields) { +static void SendFields(const std::vector & fields) { for(auto* field : fields) { field->Send(); } } -static void CopyFields(const std::vector & from_fields, - const std::vector & to_fields) { +static void CopyFields(const std::vector & from_fields, + const std::vector & to_fields) { PCMS_ALWAYS_ASSERT(from_fields.size() == to_fields.size()); for(size_t i=0; iGetFieldAdapter>(); @@ -79,8 +79,8 @@ static void AverageAndSetField(const pcms::OmegaHField & a, pcms::OmegaHField * Takes the average of each pair of fields and sets the results in the the second * argument */ -static void AverageAndSetFields(const std::vector & from_fields, - const std::vector & to_fields) { +static void AverageAndSetFields(const std::vector & from_fields, + const std::vector & to_fields) { PCMS_ALWAYS_ASSERT(from_fields.size() == to_fields.size()); for(size_t i=0; iGetFieldAdapter>(); From be72c1fef2f22aa0a7e38905779069f751b73609 Mon Sep 17 00:00:00 2001 From: Angelyr Date: Fri, 28 Mar 2025 14:41:43 -0700 Subject: [PATCH 07/12] removed name field --- src/pcms/coupler.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pcms/coupler.h b/src/pcms/coupler.h index c52fc00e..fb1c1bc3 100644 --- a/src/pcms/coupler.h +++ b/src/pcms/coupler.h @@ -128,10 +128,9 @@ class Coupler adios2::Params params = {{"Streaming", "On"}, {"OpenTimeoutSecs", "60"}}, std::string path = "") - : name_(std::move(name)), - mpi_comm_(comm), + : mpi_comm_(comm), redev_(SetUpRedev(isServer, std::move(partition))), - channel_{redev_.CreateAdiosChannel(name_, std::move(params), + channel_{redev_.CreateAdiosChannel(std::move(name), std::move(params), transport_type, std::move(path))} { PCMS_FUNCTION_TIMER; @@ -225,7 +224,6 @@ class Coupler } private: - std::string name_; MPI_Comm mpi_comm_; redev::Redev redev_; // map rather than unordered_map is necessary to avoid iterator invalidation. From 85ec0764c53d66e058e994a7daf0fd36b0db3346 Mon Sep 17 00:00:00 2001 From: Angelyr Date: Fri, 28 Mar 2025 20:01:31 -0700 Subject: [PATCH 08/12] added set up redev to server --- src/pcms/server.h | 20 +++++++++++++------- test/test_proxy_coupling.cpp | 3 +-- test/test_proxy_coupling_xgc_server.cpp | 5 ++--- test/xgc_n0_coupling_server.cpp | 3 +-- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/pcms/server.h b/src/pcms/server.h index 032c2857..b259193d 100644 --- a/src/pcms/server.h +++ b/src/pcms/server.h @@ -10,17 +10,16 @@ namespace pcms { -// TODO: strategy to merge Server/CLient Application and Fields class Application { public: - Application(std::string name, redev::Redev& rdv, MPI_Comm comm, + Application(std::string name, MPI_Comm comm, redev::Redev& redev, adios2::Params params, redev::TransportType transport_type, std::string path) : mpi_comm_(comm), redev_(redev), - channel_{rdv.CreateAdiosChannel(std::move(name), std::move(params), + channel_{redev_.CreateAdiosChannel(std::move(name), std::move(params), transport_type, std::move(path))} { PCMS_FUNCTION_TIMER; @@ -109,23 +108,30 @@ class Application class CouplerServer { +private: + redev::Redev SetUpRedev(bool isServer, redev::Partition partition) { + if (isServer) + return redev::Redev(mpi_comm_, std::move(partition), ProcessType::Server); + else + return redev::Redev(mpi_comm_); + } public: - CouplerServer(std::string name, MPI_Comm comm, redev::Partition partition) + CouplerServer(std::string name, MPI_Comm comm, bool isServer, redev::Partition partition) : name_(std::move(name)), mpi_comm_(comm), - redev_({comm, std::move(partition), ProcessType::Server}) + redev_(SetUpRedev(isServer, std::move(partition))) { PCMS_FUNCTION_TIMER; } Application* AddApplication( std::string name, std::string path = "", redev::TransportType transport_type = redev::TransportType::BP4, - adios2::Params params = {{"Streaming", "On"}, {"OpenTimeoutSecs", "400"}}) + adios2::Params params = {{"Streaming", "On"}, {"OpenTimeoutSecs", "60"}}) { PCMS_FUNCTION_TIMER; auto key = path + name; auto [it, inserted] = applications_.template try_emplace( - key, std::move(name), redev_, mpi_comm_, redev_, + key, std::move(name), mpi_comm_, redev_, std::move(params), transport_type, std::move(path)); if (!inserted) { std::cerr << "Application with name " << name << "already exists!\n"; diff --git a/test/test_proxy_coupling.cpp b/test/test_proxy_coupling.cpp index 8fc9f173..d0983164 100644 --- a/test/test_proxy_coupling.cpp +++ b/test/test_proxy_coupling.cpp @@ -12,7 +12,6 @@ #include using pcms::Copy; -using pcms::CouplerServer; using pcms::GO; using pcms::Lagrange; using pcms::make_array_view; @@ -70,7 +69,7 @@ void xgc_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, std::string_view cpn_file) // note the xgc_coupler stores a reference to the internal mesh and it is the // user responsibility to keep it alive! pcms::CouplerServer cpl( - "proxy_couple", comm, + "proxy_couple", comm, true, redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); const auto partition = std::get(cpl.GetPartition()); auto is_overlap = diff --git a/test/test_proxy_coupling_xgc_server.cpp b/test/test_proxy_coupling_xgc_server.cpp index d3af7fc6..3c1c0bf6 100644 --- a/test/test_proxy_coupling_xgc_server.cpp +++ b/test/test_proxy_coupling_xgc_server.cpp @@ -9,7 +9,6 @@ using pcms::ConstructRCFromOmegaHMesh; using pcms::Copy; -using pcms::CouplerServer; using pcms::GO; using pcms::Lagrange; using pcms::make_array_view; @@ -27,7 +26,7 @@ void xgc_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, std::string_view cpn_file) // note the xgc_coupler stores a reference to the internal mesh and it is the // user responsibility to keep it alive! pcms::CouplerServer cpl( - "proxy_couple_server", comm, + "proxy_couple_server", comm, true, redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); const auto partition = std::get(cpl.GetPartition()); ReverseClassificationVertex rc; @@ -82,7 +81,7 @@ void omegah_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, // note the xgc_coupler stores a reference to the internal mesh and it is the // user responsibility to keep it alive! pcms::CouplerServer cpl( - "proxy_couple_server", comm, + "proxy_couple_server", comm, true, redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); const auto partition = std::get(cpl.GetPartition()); auto* application = cpl.AddApplication("proxy_couple"); diff --git a/test/xgc_n0_coupling_server.cpp b/test/xgc_n0_coupling_server.cpp index 250da3be..a19f2126 100644 --- a/test/xgc_n0_coupling_server.cpp +++ b/test/xgc_n0_coupling_server.cpp @@ -8,7 +8,6 @@ #include using pcms::Copy; -using pcms::CouplerServer; using pcms::GO; using pcms::LO; using pcms::OmegaHFieldAdapter; @@ -190,7 +189,7 @@ void omegah_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, auto time1 = std::chrono::steady_clock::now(); - pcms::CouplerServer cpl("xgc_n0_coupling", comm, + pcms::CouplerServer cpl("xgc_n0_coupling", comm, true, redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); const auto partition = std::get(cpl.GetPartition()); std::string numbering = "simNumbering"; From c53b365ac5b30be4b46293dde245427b6b041e94 Mon Sep 17 00:00:00 2001 From: Angelyr Date: Mon, 31 Mar 2025 10:32:47 -0700 Subject: [PATCH 09/12] removed old coupler from c interface --- src/pcms/capi/client.cpp | 63 +++++++++++++++++-------------- src/pcms/capi/client.h | 3 +- src/pcms/fortranapi/client_wrap.c | 4 +- src/pcms/server.h | 4 +- 4 files changed, 40 insertions(+), 34 deletions(-) diff --git a/src/pcms/capi/client.cpp b/src/pcms/capi/client.cpp index 1909871f..522245b0 100644 --- a/src/pcms/capi/client.cpp +++ b/src/pcms/capi/client.cpp @@ -2,6 +2,7 @@ #include "pcms.h" #include "pcms/xgc_field_adapter.h" #include "pcms/coupler.h" +#include "pcms/server.h" #include #include // #ifdef PCMS_HAS_OMEGA_H @@ -30,13 +31,17 @@ using FieldAdapterVariant = [[nodiscard]] PcmsClientHandle pcms_create_client(const char* name, MPI_Comm comm) { - auto* client = new pcms::Coupler(name, comm); - return {reinterpret_cast(client)}; + auto* coupler = new pcms::CouplerServer(name, comm, false, {}); + auto* app = coupler->AddApplication(name); + PcmsClientHandle handle; + handle.couplerPointer = reinterpret_cast(coupler); + handle.appPointer = reinterpret_cast(app); + return handle; } void pcms_destroy_client(PcmsClientHandle client) { - if (client.pointer != nullptr) - delete reinterpret_cast(client.pointer); + if (client.couplerPointer != nullptr) + delete reinterpret_cast(client.couplerPointer); } PcmsReverseClassificationHandle pcms_load_reverse_classification( const char* file, MPI_Comm comm) @@ -53,9 +58,9 @@ void pcms_destroy_reverse_classification(PcmsReverseClassificationHandle rc) } struct AddFieldVariantOperators { - AddFieldVariantOperators(const char* name, pcms::Coupler* client, + AddFieldVariantOperators(const char* name, pcms::Application* app, int participates) - : name_(name), client_(client), participates_(participates) + : name_(name), app_(app), participates_(participates) { } @@ -69,11 +74,11 @@ struct AddFieldVariantOperators pcms::CoupledField* operator()( const FieldAdapter& field_adapter) const noexcept { - return client_->AddField(name_, field_adapter, participates_); + return app_->AddField(name_, field_adapter, participates_); } const char* name_; - pcms::Coupler* client_; + pcms::Application* app_; bool participates_; }; @@ -84,8 +89,8 @@ PcmsFieldHandle pcms_add_field(PcmsClientHandle client_handle, const char* name, auto* adapter = reinterpret_cast(adapter_handle.pointer); - auto* client = reinterpret_cast(client_handle.pointer); - PCMS_ALWAYS_ASSERT(client != nullptr); + auto* app = reinterpret_cast(client_handle.appPointer); + PCMS_ALWAYS_ASSERT(app != nullptr); PCMS_ALWAYS_ASSERT(adapter != nullptr); // pcms::CoupledField* field = std::visit( // redev::overloaded{ @@ -95,20 +100,20 @@ PcmsFieldHandle pcms_add_field(PcmsClientHandle client_handle, const char* name, // }}, // *adapter); pcms::CoupledField* field = - std::visit(AddFieldVariantOperators{name, client, participates}, *adapter); + std::visit(AddFieldVariantOperators{name, app, participates}, *adapter); return {reinterpret_cast(field)}; } void pcms_send_field_name(PcmsClientHandle client_handle, const char* name) { - auto* client = reinterpret_cast(client_handle.pointer); - PCMS_ALWAYS_ASSERT(client != nullptr); - client->SendField(name); + auto* app = reinterpret_cast(client_handle.appPointer); + PCMS_ALWAYS_ASSERT(app != nullptr); + app->SendField(name); } void pcms_receive_field_name(PcmsClientHandle client_handle, const char* name) { - auto* client = reinterpret_cast(client_handle.pointer); - PCMS_ALWAYS_ASSERT(client != nullptr); - client->ReceiveField(name); + auto* app = reinterpret_cast(client_handle.appPointer); + PCMS_ALWAYS_ASSERT(app != nullptr); + app->ReceiveField(name); } void pcms_send_field(PcmsFieldHandle field_handle) { @@ -199,25 +204,25 @@ int pcms_reverse_classification_count_verts(PcmsReverseClassificationHandle rc) } void pcms_begin_send_phase(PcmsClientHandle h) { - auto* client = reinterpret_cast(h.pointer); - PCMS_ALWAYS_ASSERT(client != nullptr); - client->BeginSendPhase(); + auto* app = reinterpret_cast(h.appPointer); + PCMS_ALWAYS_ASSERT(app != nullptr); + app->BeginSendPhase(); } void pcms_end_send_phase(PcmsClientHandle h) { - auto* client = reinterpret_cast(h.pointer); - PCMS_ALWAYS_ASSERT(client != nullptr); - client->EndSendPhase(); + auto* app = reinterpret_cast(h.appPointer); + PCMS_ALWAYS_ASSERT(app != nullptr); + app->EndSendPhase(); } void pcms_begin_receive_phase(PcmsClientHandle h) { - auto* client = reinterpret_cast(h.pointer); - PCMS_ALWAYS_ASSERT(client != nullptr); - client->BeginReceivePhase(); + auto* app = reinterpret_cast(h.appPointer); + PCMS_ALWAYS_ASSERT(app != nullptr); + app->BeginReceivePhase(); } void pcms_end_receive_phase(PcmsClientHandle h) { - auto* client = reinterpret_cast(h.pointer); - PCMS_ALWAYS_ASSERT(client != nullptr); - client->EndReceivePhase(); + auto* app = reinterpret_cast(h.appPointer); + PCMS_ALWAYS_ASSERT(app != nullptr); + app->EndReceivePhase(); } \ No newline at end of file diff --git a/src/pcms/capi/client.h b/src/pcms/capi/client.h index e03a3332..5aa8eec6 100644 --- a/src/pcms/capi/client.h +++ b/src/pcms/capi/client.h @@ -9,7 +9,8 @@ extern "C" { struct PcmsClientHandle { - void* pointer; + void* couplerPointer; + void* appPointer; }; typedef struct PcmsClientHandle PcmsClientHandle; struct PcmsOmegaHMeshHandle diff --git a/src/pcms/fortranapi/client_wrap.c b/src/pcms/fortranapi/client_wrap.c index 9b6275a8..f92bf680 100644 --- a/src/pcms/fortranapi/client_wrap.c +++ b/src/pcms/fortranapi/client_wrap.c @@ -277,7 +277,7 @@ SWIGEXPORT void _wrap_PcmsClientHandle_pointer_set(SwigClassWrapper* farg1, arg1 = (struct PcmsClientHandle*)farg1->cptr; arg2 = (void*)(*farg2); if (arg1) - (arg1)->pointer = arg2; + (arg1)->appPointer = arg2; } SWIGEXPORT void* _wrap_PcmsClientHandle_pointer_get(SwigClassWrapper* farg1) @@ -289,7 +289,7 @@ SWIGEXPORT void* _wrap_PcmsClientHandle_pointer_get(SwigClassWrapper* farg1) SWIG_check_nonnull(farg1->cptr, "struct PcmsClientHandle *", "PcmsClientHandle", "PcmsClientHandle::pointer", return 0); arg1 = (struct PcmsClientHandle*)farg1->cptr; - result = (void*)((arg1)->pointer); + result = (void*)((arg1)->appPointer); fresult = (void*)(result); return fresult; } diff --git a/src/pcms/server.h b/src/pcms/server.h index b259193d..44c60ddd 100644 --- a/src/pcms/server.h +++ b/src/pcms/server.h @@ -28,12 +28,12 @@ class Application // These fields are supposed to be agnostic to adios2... template CoupledField* AddField( - std::string name, FieldAdapterT&& field_adapter) + std::string name, FieldAdapterT&& field_adapter, bool participates = true) { PCMS_FUNCTION_TIMER; auto [it, inserted] = fields_.template try_emplace( name, name, std::forward(field_adapter), mpi_comm_, redev_, - channel_); + channel_, participates); if (!inserted) { std::cerr << "Field with this name" << name << "already exists!\n"; std::terminate(); From 3b05afc42118c25faabc324968e2018a3dabbf6f Mon Sep 17 00:00:00 2001 From: Angelyr Date: Wed, 2 Apr 2025 10:23:05 -0700 Subject: [PATCH 10/12] removed server --- src/CMakeLists.txt | 6 - src/pcms.h | 10 +- src/pcms/capi/client.cpp | 6 +- src/pcms/coupler.h | 112 ++++++++++------- src/pcms/server.h | 156 ------------------------ test/test_proxy_coupling.cpp | 41 ++++--- test/test_proxy_coupling_xgc_server.cpp | 4 +- test/xgc_n0_coupling_server.cpp | 2 +- 8 files changed, 101 insertions(+), 236 deletions(-) delete mode 100644 src/pcms/server.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 35bb7ad9..acfa75d5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -44,12 +44,6 @@ if (PCMS_ENABLE_OMEGA_H) pcms/uniform_grid.h pcms/point_search.h) endif () -if (PCMS_ENABLE_SERVER) - if (NOT PCMS_ENABLE_OMEGA_H) - message(ERROR "PCMS_ENABLE_OMEGA_H is required for server implementation") - endif () - list(APPEND PCMS_HEADERS pcms/server.h) -endif () find_package(Kokkos REQUIRED) find_package(perfstubs REQUIRED) diff --git a/src/pcms.h b/src/pcms.h index 14eb2241..1c046332 100644 --- a/src/pcms.h +++ b/src/pcms.h @@ -1,8 +1,12 @@ #ifndef PCMS_H_ #define PCMS_H_ -#ifdef PCMS_HAS_SERVER -#include "pcms/server.h" -#endif +#include "pcms/common.h" +#include "pcms/field_communicator.h" +#include "pcms/omega_h_field.h" +#include "pcms/profile.h" +#include "pcms/coupler.h" +#include +#include #endif diff --git a/src/pcms/capi/client.cpp b/src/pcms/capi/client.cpp index 522245b0..85d59ce0 100644 --- a/src/pcms/capi/client.cpp +++ b/src/pcms/capi/client.cpp @@ -1,8 +1,6 @@ #include "client.h" #include "pcms.h" #include "pcms/xgc_field_adapter.h" -#include "pcms/coupler.h" -#include "pcms/server.h" #include #include // #ifdef PCMS_HAS_OMEGA_H @@ -31,7 +29,7 @@ using FieldAdapterVariant = [[nodiscard]] PcmsClientHandle pcms_create_client(const char* name, MPI_Comm comm) { - auto* coupler = new pcms::CouplerServer(name, comm, false, {}); + auto* coupler = new pcms::Coupler(name, comm, false, {}); auto* app = coupler->AddApplication(name); PcmsClientHandle handle; handle.couplerPointer = reinterpret_cast(coupler); @@ -41,7 +39,7 @@ using FieldAdapterVariant = void pcms_destroy_client(PcmsClientHandle client) { if (client.couplerPointer != nullptr) - delete reinterpret_cast(client.couplerPointer); + delete reinterpret_cast(client.couplerPointer); } PcmsReverseClassificationHandle pcms_load_reverse_classification( const char* file, MPI_Comm comm) diff --git a/src/pcms/coupler.h b/src/pcms/coupler.h index fb1c1bc3..45b0db2f 100644 --- a/src/pcms/coupler.h +++ b/src/pcms/coupler.h @@ -112,67 +112,42 @@ class CoupledField std::unique_ptr coupled_field_; }; -class Coupler +class Application { -private: - redev::Redev SetUpRedev(bool isServer, redev::Partition partition) { - if (isServer) - return redev::Redev(mpi_comm_, std::move(partition)); - else - return redev::Redev(mpi_comm_); - } public: - Coupler(std::string name, MPI_Comm comm, bool isServer = false, - redev::Partition partition = redev::Partition{redev::RCBPtn()}, - redev::TransportType transport_type = redev::TransportType::BP4, - adios2::Params params = {{"Streaming", "On"}, - {"OpenTimeoutSecs", "60"}}, - std::string path = "") + Application(std::string name, MPI_Comm comm, + redev::Redev& redev, adios2::Params params, + redev::TransportType transport_type, + std::string path) : mpi_comm_(comm), - redev_(SetUpRedev(isServer, std::move(partition))), + redev_(redev), channel_{redev_.CreateAdiosChannel(std::move(name), std::move(params), - transport_type, std::move(path))} - { - PCMS_FUNCTION_TIMER; - } - - [[nodiscard]] const redev::Partition& GetPartition() const + transport_type, std::move(path))} { PCMS_FUNCTION_TIMER; - return redev_.GetPartition(); } - - /** - * @known_issue - * The redev partion has to be same as the partition of the OH mesh. - * It asserts the number of elements sent and received are same. - * otherwise, ConstructPermutation() will fail. - */ + // FIXME should take a file path for the parameters, not take adios2 params. + // These fields are supposed to be agnostic to adios2... template - CoupledField* AddField(std::string name, FieldAdapterT field_adapter, - bool participates = true) + CoupledField* AddField( + std::string name, FieldAdapterT&& field_adapter, bool participates = true) { PCMS_FUNCTION_TIMER; - auto [it, inserted] = - fields_.template try_emplace(name, name, std::move(field_adapter), - mpi_comm_, redev_, channel_, participates); + auto [it, inserted] = fields_.template try_emplace( + name, name, std::forward(field_adapter), mpi_comm_, redev_, + channel_, participates); if (!inserted) { - std::cerr << "OHField with this name" << name << "already exists!\n"; + std::cerr << "Field with this name" << name << "already exists!\n"; std::terminate(); } return &(it->second); } - - // take a string& since map cannot be searched with string_view - // (heterogeneous lookup) void SendField(const std::string& name, Mode mode = Mode::Synchronous) { PCMS_FUNCTION_TIMER; PCMS_ALWAYS_ASSERT(InSendPhase()); detail::find_or_error(name, fields_).Send(mode); }; - // take a string& since map cannot be searched with string_view - // (heterogeneous lookup) void ReceiveField(const std::string& name, Mode mode = Mode::Synchronous) { PCMS_FUNCTION_TIMER; @@ -225,12 +200,61 @@ class Coupler private: MPI_Comm mpi_comm_; - redev::Redev redev_; - // map rather than unordered_map is necessary to avoid iterator invalidation. - // This is important because we pass pointers to the fields out of this class - std::map fields_; + redev::Redev& redev_; redev::Channel channel_; + // map is used rather than unordered_map because we give pointers to the + // internal data and rehash of unordered_map can cause pointer invalidation. + // map is less cache friendly, but pointers are not invalidated. + std::map fields_; }; + +class Coupler +{ +private: + redev::Redev SetUpRedev(bool isServer, redev::Partition partition) { + if (isServer) + return redev::Redev(mpi_comm_, std::move(partition), ProcessType::Server); + else + return redev::Redev(mpi_comm_); + } +public: + Coupler(std::string name, MPI_Comm comm, bool isServer, redev::Partition partition) + : name_(std::move(name)), + mpi_comm_(comm), + redev_(SetUpRedev(isServer, std::move(partition))) + { + PCMS_FUNCTION_TIMER; + } + Application* AddApplication( + std::string name, std::string path = "", + redev::TransportType transport_type = redev::TransportType::BP4, + adios2::Params params = {{"Streaming", "On"}, {"OpenTimeoutSecs", "60"}}) + { + PCMS_FUNCTION_TIMER; + auto key = path + name; + auto [it, inserted] = applications_.template try_emplace( + key, std::move(name), mpi_comm_, redev_, + std::move(params), transport_type, std::move(path)); + if (!inserted) { + std::cerr << "Application with name " << name << "already exists!\n"; + std::terminate(); + } + return &(it->second); + } + + [[nodiscard]] const redev::Partition& GetPartition() const noexcept + { + return redev_.GetPartition(); + } + +private: + std::string name_; + MPI_Comm mpi_comm_; + redev::Redev redev_; + // gather and scatter operations have reference to internal fields + std::map applications_; +}; + } // namespace pcms #endif // PCMS_COUPLER_H diff --git a/src/pcms/server.h b/src/pcms/server.h deleted file mode 100644 index 44c60ddd..00000000 --- a/src/pcms/server.h +++ /dev/null @@ -1,156 +0,0 @@ -#ifndef PCMS_COUPLING_SERVER_H -#define PCMS_COUPLING_SERVER_H -#include "pcms/common.h" -#include "pcms/field_communicator.h" -#include "pcms/omega_h_field.h" -#include "pcms/profile.h" -#include "pcms/coupler.h" -#include -#include - -namespace pcms -{ -class Application -{ -public: - Application(std::string name, MPI_Comm comm, - redev::Redev& redev, adios2::Params params, - redev::TransportType transport_type, - std::string path) - : mpi_comm_(comm), - redev_(redev), - channel_{redev_.CreateAdiosChannel(std::move(name), std::move(params), - transport_type, std::move(path))} - { - PCMS_FUNCTION_TIMER; - } - // FIXME should take a file path for the parameters, not take adios2 params. - // These fields are supposed to be agnostic to adios2... - template - CoupledField* AddField( - std::string name, FieldAdapterT&& field_adapter, bool participates = true) - { - PCMS_FUNCTION_TIMER; - auto [it, inserted] = fields_.template try_emplace( - name, name, std::forward(field_adapter), mpi_comm_, redev_, - channel_, participates); - if (!inserted) { - std::cerr << "Field with this name" << name << "already exists!\n"; - std::terminate(); - } - return &(it->second); - } - void SendField(const std::string& name, Mode mode = Mode::Synchronous) - { - PCMS_FUNCTION_TIMER; - PCMS_ALWAYS_ASSERT(InSendPhase()); - detail::find_or_error(name, fields_).Send(mode); - }; - void ReceiveField(const std::string& name, Mode mode = Mode::Synchronous) - { - PCMS_FUNCTION_TIMER; - PCMS_ALWAYS_ASSERT(InReceivePhase()); - detail::find_or_error(name, fields_).Receive(mode); - }; - [[nodiscard]] bool InSendPhase() const noexcept - { - PCMS_FUNCTION_TIMER; - return channel_.InSendCommunicationPhase(); - } - [[nodiscard]] bool InReceivePhase() const noexcept - { - PCMS_FUNCTION_TIMER; - return channel_.InReceiveCommunicationPhase(); - } - void BeginSendPhase() - { - PCMS_FUNCTION_TIMER; - channel_.BeginSendCommunicationPhase(); - } - void EndSendPhase() - { - PCMS_FUNCTION_TIMER; - channel_.EndSendCommunicationPhase(); - } - void BeginReceivePhase() - { - PCMS_FUNCTION_TIMER; - channel_.BeginReceiveCommunicationPhase(); - } - void EndReceivePhase() - { - PCMS_FUNCTION_TIMER; - channel_.EndReceiveCommunicationPhase(); - } - - template - auto SendPhase(const Func& func, Args&&... args) - { - PCMS_FUNCTION_TIMER; - return channel_.SendPhase(func, std::forward(args)...); - } - template - auto ReceivePhase(const Func& func, Args&&... args) - { - PCMS_FUNCTION_TIMER; - return channel_.ReceivePhase(func, std::forward(args)...); - } - -private: - MPI_Comm mpi_comm_; - redev::Redev& redev_; - redev::Channel channel_; - // map is used rather than unordered_map because we give pointers to the - // internal data and rehash of unordered_map can cause pointer invalidation. - // map is less cache friendly, but pointers are not invalidated. - std::map fields_; -}; - -class CouplerServer -{ -private: - redev::Redev SetUpRedev(bool isServer, redev::Partition partition) { - if (isServer) - return redev::Redev(mpi_comm_, std::move(partition), ProcessType::Server); - else - return redev::Redev(mpi_comm_); - } -public: - CouplerServer(std::string name, MPI_Comm comm, bool isServer, redev::Partition partition) - : name_(std::move(name)), - mpi_comm_(comm), - redev_(SetUpRedev(isServer, std::move(partition))) - { - PCMS_FUNCTION_TIMER; - } - Application* AddApplication( - std::string name, std::string path = "", - redev::TransportType transport_type = redev::TransportType::BP4, - adios2::Params params = {{"Streaming", "On"}, {"OpenTimeoutSecs", "60"}}) - { - PCMS_FUNCTION_TIMER; - auto key = path + name; - auto [it, inserted] = applications_.template try_emplace( - key, std::move(name), mpi_comm_, redev_, - std::move(params), transport_type, std::move(path)); - if (!inserted) { - std::cerr << "Application with name " << name << "already exists!\n"; - std::terminate(); - } - return &(it->second); - } - - [[nodiscard]] const redev::Partition& GetPartition() const noexcept - { - return redev_.GetPartition(); - } - -private: - std::string name_; - MPI_Comm mpi_comm_; - redev::Redev redev_; - // gather and scatter operations have reference to internal fields - std::map applications_; -}; -} // namespace pcms -#endif // PCMS_COUPLING_SERVER_H diff --git a/test/test_proxy_coupling.cpp b/test/test_proxy_coupling.cpp index d0983164..5aead8f0 100644 --- a/test/test_proxy_coupling.cpp +++ b/test/test_proxy_coupling.cpp @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -26,40 +25,42 @@ namespace ts = test_support; void xgc_delta_f(MPI_Comm comm, Omega_h::Mesh& mesh) { - pcms::Coupler cpl("proxy_couple_xgc_delta_f", comm); + pcms::Coupler coupler("proxy_couple", comm, false, {}); + pcms::Application* app = coupler.AddApplication("proxy_couple_xgc_delta_f"); auto is_overlap = ts::markOverlapMeshEntities(mesh, ts::IsModelEntInOverlap{}); - cpl.AddField("gids", + app->AddField("gids", OmegaHFieldAdapter("global", mesh, is_overlap)); - cpl.AddField("gids2", + app->AddField("gids2", OmegaHFieldAdapter("global", mesh, is_overlap)); do { for (int i = 0; i < COMM_ROUNDS; ++i) { - cpl.BeginSendPhase(); - cpl.SendField("gids"); //(Alt) df_gid_field->Send(); - cpl.SendField("gids2"); //(Alt) df_gid_field->Send(); - cpl.EndSendPhase(); - cpl.BeginReceivePhase(); - cpl.ReceiveField("gids"); //(Alt) df_gid_field->Receive(); - cpl.EndReceivePhase(); + app->BeginSendPhase(); + app->SendField("gids"); //(Alt) df_gid_field->Send(); + app->SendField("gids2"); //(Alt) df_gid_field->Send(); + app->EndSendPhase(); + app->BeginReceivePhase(); + app->ReceiveField("gids"); //(Alt) df_gid_field->Receive(); + app->EndReceivePhase(); // cpl.ReceiveField("gids2"); //(Alt) df_gid_field->Receive(); } } while (!done); } void xgc_total_f(MPI_Comm comm, Omega_h::Mesh& mesh) { - pcms::Coupler cpl("proxy_couple_xgc_total_f", comm); + pcms::Coupler coupler("proxy_couple", comm, false, {}); + pcms::Application* app = coupler.AddApplication("proxy_couple_xgc_total_f"); auto is_overlap = ts::markOverlapMeshEntities(mesh, ts::IsModelEntInOverlap{}); - cpl.AddField("gids", + app->AddField("gids", OmegaHFieldAdapter("global", mesh, is_overlap)); do { for (int i = 0; i < COMM_ROUNDS; ++i) { - cpl.BeginSendPhase(); - cpl.SendField("gids"); //(Alt) tf_gid_field->Send(); - cpl.EndSendPhase(); - cpl.BeginReceivePhase(); - cpl.ReceiveField("gids"); //(Alt) tf_gid_field->Receive(); - cpl.EndReceivePhase(); + app->BeginSendPhase(); + app->SendField("gids"); //(Alt) tf_gid_field->Send(); + app->EndSendPhase(); + app->BeginReceivePhase(); + app->ReceiveField("gids"); //(Alt) tf_gid_field->Receive(); + app->EndReceivePhase(); } } while (!done); } @@ -68,7 +69,7 @@ void xgc_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, std::string_view cpn_file) // coupling server using same mesh as application // note the xgc_coupler stores a reference to the internal mesh and it is the // user responsibility to keep it alive! - pcms::CouplerServer cpl( + pcms::Coupler cpl( "proxy_couple", comm, true, redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); const auto partition = std::get(cpl.GetPartition()); diff --git a/test/test_proxy_coupling_xgc_server.cpp b/test/test_proxy_coupling_xgc_server.cpp index 3c1c0bf6..69039348 100644 --- a/test/test_proxy_coupling_xgc_server.cpp +++ b/test/test_proxy_coupling_xgc_server.cpp @@ -25,7 +25,7 @@ void xgc_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, std::string_view cpn_file) // coupling server using same mesh as application // note the xgc_coupler stores a reference to the internal mesh and it is the // user responsibility to keep it alive! - pcms::CouplerServer cpl( + pcms::Coupler cpl( "proxy_couple_server", comm, true, redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); const auto partition = std::get(cpl.GetPartition()); @@ -80,7 +80,7 @@ void omegah_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, // coupling server using same mesh as application // note the xgc_coupler stores a reference to the internal mesh and it is the // user responsibility to keep it alive! - pcms::CouplerServer cpl( + pcms::Coupler cpl( "proxy_couple_server", comm, true, redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); const auto partition = std::get(cpl.GetPartition()); diff --git a/test/xgc_n0_coupling_server.cpp b/test/xgc_n0_coupling_server.cpp index a19f2126..2b1d1853 100644 --- a/test/xgc_n0_coupling_server.cpp +++ b/test/xgc_n0_coupling_server.cpp @@ -189,7 +189,7 @@ void omegah_coupler(MPI_Comm comm, Omega_h::Mesh& mesh, auto time1 = std::chrono::steady_clock::now(); - pcms::CouplerServer cpl("xgc_n0_coupling", comm, true, + pcms::Coupler cpl("xgc_n0_coupling", comm, true, redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); const auto partition = std::get(cpl.GetPartition()); std::string numbering = "simNumbering"; From 63dafd5e23485f70b656fe641e516d38c858866d Mon Sep 17 00:00:00 2001 From: Angelyr Date: Wed, 2 Apr 2025 10:38:26 -0700 Subject: [PATCH 11/12] remove unused --- src/pcms.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pcms.h b/src/pcms.h index 1c046332..e39c3612 100644 --- a/src/pcms.h +++ b/src/pcms.h @@ -6,7 +6,5 @@ #include "pcms/omega_h_field.h" #include "pcms/profile.h" #include "pcms/coupler.h" -#include -#include #endif From ab57593eacd31b9a66ed6bedea382948b62b1671 Mon Sep 17 00:00:00 2001 From: Angelyr Date: Fri, 4 Apr 2025 18:08:19 -0700 Subject: [PATCH 12/12] update version --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5232858c..2df612b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.19) -project(pcms VERSION 0.1.2 LANGUAGES CXX) +project(pcms VERSION 0.2.0 LANGUAGES CXX) include(GNUInstallDirs) include(CMakePackageConfigHelpers)