From 87ce8439040947da041c8ca1630fff1ff9c2eb61 Mon Sep 17 00:00:00 2001 From: Angelyr Date: Wed, 29 Jan 2025 17:05:25 -0500 Subject: [PATCH 1/6] created partition wrapper --- src/CMakeLists.txt | 1 + src/pcms/omega_h_field.h | 28 +++------------------ src/pcms/partition.h | 47 ++++++++++++++++++++++++++++++++++++ src/pcms/xgc_field_adapter.h | 27 ++------------------- 4 files changed, 53 insertions(+), 50 deletions(-) create mode 100644 src/pcms/partition.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8b249374..b9c079d7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -17,6 +17,7 @@ set(PCMS_HEADERS pcms/inclusive_scan.h pcms/profile.h pcms/print.h + pcms/partition.h ) set(PCMS_SOURCES diff --git a/src/pcms/omega_h_field.h b/src/pcms/omega_h_field.h index 2ceb6c60..36963a7e 100644 --- a/src/pcms/omega_h_field.h +++ b/src/pcms/omega_h_field.h @@ -17,6 +17,7 @@ #include "pcms/transfer_field.h" #include "pcms/memory_spaces.h" #include "pcms/profile.h" +#include "pcms/partition.h" // FIXME add executtion spaces (don't use kokkos exe spaces directly) @@ -85,29 +86,6 @@ Omega_h::Read filter_array(Omega_h::Read array, }); return filtered_field; } -struct GetRankOmegaH -{ - GetRankOmegaH(int i, Omega_h::I8 dim, Omega_h::ClassId id, std::array & coord) - : i_(i), id_(id), dim_(dim), coord_(coord) - { - PCMS_FUNCTION_TIMER; - } - auto operator()(const redev::ClassPtn& ptn) const - { - PCMS_FUNCTION_TIMER; - const auto ent = redev::ClassPtn::ModelEnt({dim_, id_}); - return ptn.GetRank(ent); - } - auto operator()(const redev::RCBPtn& ptn) - { - PCMS_FUNCTION_TIMER; - return ptn.GetRank(coord_); - } - int i_; - Omega_h::ClassId id_; - Omega_h::I8 dim_; - std::array coord_; -}; } // namespace detail template coord; pcms::ReversePartitionMap reverse_partition; pcms::LO local_index = 0; + Partition part{partition}; for (auto i = 0; i < classIds_h.size(); i++) { coord[0] = coords[i * dim]; coord[1] = coords[i * dim + 1]; coord[2] = (dim == 3) ? coords[i * dim + 2] : 0.0; - auto dr = std::visit(detail::GetRankOmegaH{i, classDims_h[i], classIds_h[i], coord}, - partition); + auto dr = part.GetDr(classIds_h[i], classDims_h[i], coord); reverse_partition[dr].emplace_back(local_index++); } return reverse_partition; diff --git a/src/pcms/partition.h b/src/pcms/partition.h new file mode 100644 index 00000000..d9a3c8c6 --- /dev/null +++ b/src/pcms/partition.h @@ -0,0 +1,47 @@ +#ifndef PCMS_PARTITION_H +#define PCMS_PARTITION_H +#include "pcms/common.h" + + +namespace pcms +{ + struct GetRank + { + GetRank(LO id, LO dim, std::array& coord) + : id_(id), dim_(dim), coord_(coord) + { + PCMS_FUNCTION_TIMER; + } + auto operator()(const redev::ClassPtn& ptn) const + { + PCMS_FUNCTION_TIMER; + const auto ent = redev::ClassPtn::ModelEnt({dim_, id_}); + return ptn.GetRank(ent); + } + auto operator()(const redev::RCBPtn& ptn) + { + PCMS_FUNCTION_TIMER; + return ptn.GetRank(coord_); + } + LO id_; + LO dim_; + std::array coord_; + }; + + struct Partition + { + Partition(const redev::Partition& partition) : partition_(partition) + { + PCMS_FUNCTION_TIMER; + } + + auto GetDr(LO id, LO dim, std::array coord = {}) + { + return std::visit(GetRank{id, dim, coord}, partition_); + } + + redev::Partition partition_; + }; +} // namespace pcms + +#endif //PCMS_PARTITION_H \ No newline at end of file diff --git a/src/pcms/xgc_field_adapter.h b/src/pcms/xgc_field_adapter.h index 354f12a9..3b49bfe9 100644 --- a/src/pcms/xgc_field_adapter.h +++ b/src/pcms/xgc_field_adapter.h @@ -13,30 +13,6 @@ namespace pcms { -namespace detail -{ -// Needed since NVHPC doesn't work with overloaded -struct GetRank -{ - using GeomType = DimID; - GetRank(const GeomType& geom) : geom_(geom) {} - auto operator()(const redev::ClassPtn& ptn) const - { - PCMS_FUNCTION_TIMER; - const auto ent = redev::ClassPtn::ModelEnt({geom_.dim, geom_.id}); - return ptn.GetRank(ent); - } - auto operator()(const redev::RCBPtn& /*unused*/) const - { - PCMS_FUNCTION_TIMER; - std::cerr << "RCB partition not handled yet\n"; - std::terminate(); - return 0; - } - const GeomType& geom_; -}; -} // namespace detail - template class XGCFieldAdapter { @@ -144,11 +120,12 @@ class XGCFieldAdapter pcms::ReversePartitionMap reverse_partition; // in_overlap_ must contain a function! PCMS_ALWAYS_ASSERT(static_cast(in_overlap_)); + Partition part{partition}; for (const auto& geom : reverse_classification_) { // if the geometry is in specified overlap region if (in_overlap_(geom.first.dim, geom.first.id)) { - auto dr = std::visit(detail::GetRank{geom.first}, partition); + auto dr = part.GetDr(geom.first.id, geom.first.dim); auto [it, inserted] = reverse_partition.try_emplace(dr); // the map gives the local iteration order of the global ids auto map = mask_.GetMap(); From ec90f79ba864bfc8e5b6ea3de3a1f95ee3e52c45 Mon Sep 17 00:00:00 2001 From: Angelyr Date: Wed, 29 Jan 2025 17:10:38 -0500 Subject: [PATCH 2/6] update version --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a0f4c4f5..f5f56df8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.19) -project(pcms VERSION 0.1.1 LANGUAGES CXX) +project(pcms VERSION 0.1.2 LANGUAGES CXX) include(GNUInstallDirs) include(CMakePackageConfigHelpers) From b3807f5cff250b43be90f10cd932007f2c186189 Mon Sep 17 00:00:00 2001 From: Angelyr Date: Fri, 31 Jan 2025 19:28:01 -0500 Subject: [PATCH 3/6] changed change reverse partition map signiture --- src/pcms/dummy_field_adapter.h | 2 +- src/pcms/field_communicator.h | 4 +++- src/pcms/omega_h_field.h | 5 ++--- src/pcms/partition.h | 4 ++-- src/pcms/xgc_field_adapter.h | 5 ++--- test/test_ohClassPtn_appRibPtn.cpp | 2 -- 6 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/pcms/dummy_field_adapter.h b/src/pcms/dummy_field_adapter.h index 8a2110c1..2bbdabff 100644 --- a/src/pcms/dummy_field_adapter.h +++ b/src/pcms/dummy_field_adapter.h @@ -8,7 +8,7 @@ class DummyFieldAdapter using value_type = int; [[nodiscard]] std::vector GetGids() const { return {}; } [[nodiscard]] ReversePartitionMap GetReversePartitionMap( - const redev::Partition& partition) const + const Partition& partition) const { return {}; } diff --git a/src/pcms/field_communicator.h b/src/pcms/field_communicator.h index d45b1573..4b52448a 100644 --- a/src/pcms/field_communicator.h +++ b/src/pcms/field_communicator.h @@ -5,6 +5,8 @@ #include #include "pcms/inclusive_scan.h" #include "pcms/profile.h" +#include "pcms/partition.h" + namespace pcms { @@ -203,7 +205,7 @@ struct FieldCommunicator auto gids = field_adapter_.GetGids(); if (redev_.GetProcessType() == redev::ProcessType::Client) { const ReversePartitionMap reverse_partition = - field_adapter_.GetReversePartitionMap(redev_.GetPartition()); + field_adapter_.GetReversePartitionMap(Partition{redev_.GetPartition()}); auto out_message = ConstructOutMessage(reverse_partition); comm_.SetOutMessageLayout(out_message.dest, out_message.offset); gid_comm_.SetOutMessageLayout(out_message.dest, out_message.offset); diff --git a/src/pcms/omega_h_field.h b/src/pcms/omega_h_field.h index 36963a7e..3f09898a 100644 --- a/src/pcms/omega_h_field.h +++ b/src/pcms/omega_h_field.h @@ -535,7 +535,7 @@ class OmegaHFieldAdapter } // REQUIRED [[nodiscard]] ReversePartitionMap GetReversePartitionMap( - const redev::Partition& partition) const + const Partition& partition) const { PCMS_FUNCTION_TIMER; auto classIds_h = Omega_h::HostRead(field_.GetClassIDs()); @@ -549,12 +549,11 @@ class OmegaHFieldAdapter std::array coord; pcms::ReversePartitionMap reverse_partition; pcms::LO local_index = 0; - Partition part{partition}; for (auto i = 0; i < classIds_h.size(); i++) { coord[0] = coords[i * dim]; coord[1] = coords[i * dim + 1]; coord[2] = (dim == 3) ? coords[i * dim + 2] : 0.0; - auto dr = part.GetDr(classIds_h[i], classDims_h[i], coord); + auto dr = partition.GetDr(classIds_h[i], classDims_h[i], coord); reverse_partition[dr].emplace_back(local_index++); } return reverse_partition; diff --git a/src/pcms/partition.h b/src/pcms/partition.h index d9a3c8c6..d5683909 100644 --- a/src/pcms/partition.h +++ b/src/pcms/partition.h @@ -7,7 +7,7 @@ namespace pcms { struct GetRank { - GetRank(LO id, LO dim, std::array& coord) + GetRank(const LO id, const LO dim, const std::array& coord) : id_(id), dim_(dim), coord_(coord) { PCMS_FUNCTION_TIMER; @@ -35,7 +35,7 @@ namespace pcms PCMS_FUNCTION_TIMER; } - auto GetDr(LO id, LO dim, std::array coord = {}) + auto GetDr(const LO id, const LO dim, const std::array coord = {}) const { return std::visit(GetRank{id, dim, coord}, partition_); } diff --git a/src/pcms/xgc_field_adapter.h b/src/pcms/xgc_field_adapter.h index 3b49bfe9..dd8976e5 100644 --- a/src/pcms/xgc_field_adapter.h +++ b/src/pcms/xgc_field_adapter.h @@ -112,7 +112,7 @@ class XGCFieldAdapter // REQUIRED [[nodiscard]] ReversePartitionMap GetReversePartitionMap( - const redev::Partition& partition) const + const Partition& partition) const { PCMS_FUNCTION_TIMER; if (RankParticipatesCouplingCommunication()) { @@ -120,12 +120,11 @@ class XGCFieldAdapter pcms::ReversePartitionMap reverse_partition; // in_overlap_ must contain a function! PCMS_ALWAYS_ASSERT(static_cast(in_overlap_)); - Partition part{partition}; for (const auto& geom : reverse_classification_) { // if the geometry is in specified overlap region if (in_overlap_(geom.first.dim, geom.first.id)) { - auto dr = part.GetDr(geom.first.id, geom.first.dim); + auto dr = partition.GetDr(geom.first.id, geom.first.dim); auto [it, inserted] = reverse_partition.try_emplace(dr); // the map gives the local iteration order of the global ids auto map = mask_.GetMap(); diff --git a/test/test_ohClassPtn_appRibPtn.cpp b/test/test_ohClassPtn_appRibPtn.cpp index 612df09b..eb4446c9 100644 --- a/test/test_ohClassPtn_appRibPtn.cpp +++ b/test/test_ohClassPtn_appRibPtn.cpp @@ -49,8 +49,6 @@ int main(int argc, char** argv) static_cast(isRdv)); const std::string name = "meshVtxIds"; - const int rdvRanks = 2; - const int appRanks = 2; adios2::Params params{{"Streaming", "On"}, {"OpenTimeoutSecs", "60"}}; auto channel = From f63ca2906e96fcf8504f763d0729c9effd836c5b Mon Sep 17 00:00:00 2001 From: Angelyr Date: Fri, 31 Jan 2025 19:30:51 -0500 Subject: [PATCH 4/6] set ubuntu version --- .github/workflows/cmake-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake-test.yml b/.github/workflows/cmake-test.yml index 5bb60885..43d64222 100644 --- a/.github/workflows/cmake-test.yml +++ b/.github/workflows/cmake-test.yml @@ -6,7 +6,7 @@ on: jobs: test-build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 30 defaults: From f620e90a5aed6a14423853536f84d0dd603e7589 Mon Sep 17 00:00:00 2001 From: Angelyr Date: Fri, 31 Jan 2025 19:35:45 -0500 Subject: [PATCH 5/6] latest ubuntu --- .github/workflows/cmake-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake-test.yml b/.github/workflows/cmake-test.yml index 43d64222..d557bc36 100644 --- a/.github/workflows/cmake-test.yml +++ b/.github/workflows/cmake-test.yml @@ -6,7 +6,7 @@ on: jobs: test-build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 30 defaults: @@ -31,7 +31,7 @@ jobs: run: sudo apt-get update - name: Install mpi - run: sudo apt-get install -yq mpich libmpich-dev + run: sudo apt-get install -yq mpich - uses: actions/checkout@v4 From d6ad88f88365023e61950a69e02f32b44cf2137d Mon Sep 17 00:00:00 2001 From: Angelyr Date: Sat, 1 Feb 2025 12:11:57 -0800 Subject: [PATCH 6/6] updated perlmutter kokkos version --- .github/workflows/perlmutter/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/perlmutter/install.sh b/.github/workflows/perlmutter/install.sh index fca68520..ca63b2fe 100644 --- a/.github/workflows/perlmutter/install.sh +++ b/.github/workflows/perlmutter/install.sh @@ -9,7 +9,7 @@ module load cray-fftw # # kokkos # rm build-kokkos -rf # # rm kokkos -rf -# git clone -b 4.2.00 https://github.com/kokkos/kokkos.git +# git clone -b 4.5.00 https://github.com/kokkos/kokkos.git # cmake -S kokkos -B build-kokkos \ # -DCMAKE_INSTALL_PREFIX=build-kokkos/install \ # -DCMAKE_BUILD_TYPE="Release" \ @@ -100,7 +100,7 @@ cmake -S pcms -B build-pcms \ -DCMAKE_C_COMPILER=cc \ -DCMAKE_CXX_COMPILER=CC \ -DCMAKE_BUILD_TYPE=Release \ - -DPCMS_TIMEOUT=0 \ + -DPCMS_TIMEOUT=100 \ -Dperfstubs_DIR=$PWD/build-perfstubs \ -Dredev_DIR=$PWD/build-redev/install/lib64/cmake/redev \ -DOmega_h_DIR=$PWD/build-omega_h/install/lib64/cmake/Omega_h/ \