From 32f2efb9e2a97653faeb2466db0d24c6f2349034 Mon Sep 17 00:00:00 2001 From: Sergey Oblomov Date: Wed, 23 May 2018 16:50:10 +0300 Subject: [PATCH 1/2] MD/IB: added device memory allocation - added UCT flags - added DM allocation - added gtest for device memory --- src/ucp/core/ucp_mm.c | 11 ++-- src/uct/api/uct.h | 36 +++++++------ src/uct/base/uct_mem.c | 88 +++++++++++++++++++++++++++---- src/uct/ib/base/ib_md.c | 92 +++++++++++++++++++++++++++++---- src/uct/ib/base/ib_md.h | 15 +++++- src/uct/ib/mlx5/ib_mlx5_dv.h | 8 +-- test/gtest/ucp/test_ucp_mmap.cc | 82 +++++++++++++++++++++++++++++ test/gtest/uct/test_md.cc | 33 ++++++++++++ test/gtest/uct/test_mem.cc | 57 ++++++++++++++++++++ 9 files changed, 378 insertions(+), 44 deletions(-) diff --git a/src/ucp/core/ucp_mm.c b/src/ucp/core/ucp_mm.c index 8f2bc75f5c3..d20df0ed9df 100644 --- a/src/ucp/core/ucp_mm.c +++ b/src/ucp/core/ucp_mm.c @@ -219,7 +219,13 @@ ucp_mem_map_params2uct_flags(ucp_mem_map_params_t *params) } if (params->flags & UCP_MEM_MAP_FIXED) { - flags |= UCT_MD_MEM_FLAG_FIXED; + if ((params->field_mask & UCP_MEM_MAP_PARAM_FIELD_ADDRESS) && + (params->address == NULL)) { + /* special hint: allocate memory on device */ + flags |= UCT_MD_MEM_FLAG_ON_DEVICE; + } else { + flags |= UCT_MD_MEM_FLAG_FIXED; + } } } @@ -261,8 +267,7 @@ static inline ucs_status_t ucp_mem_map_check_and_adjust_params(ucp_mem_map_param } if ((params->flags & UCP_MEM_MAP_FIXED) && - (!params->address || - ((uintptr_t)params->address % ucs_get_page_size()))) { + ((uintptr_t)params->address % ucs_get_page_size())) { ucs_error("UCP_MEM_MAP_FIXED flag requires page aligned address"); return UCS_ERR_INVALID_PARAM; } diff --git a/src/uct/api/uct.h b/src/uct/api/uct.h index 41634dcce6b..f5dd740867e 100644 --- a/src/uct/api/uct.h +++ b/src/uct/api/uct.h @@ -509,22 +509,23 @@ typedef enum { * @brief Memory domain capability flags. */ enum { - UCT_MD_FLAG_ALLOC = UCS_BIT(0), /**< MD supports memory allocation */ - UCT_MD_FLAG_REG = UCS_BIT(1), /**< MD supports memory registration */ - UCT_MD_FLAG_NEED_MEMH = UCS_BIT(2), /**< The transport needs a valid local - memory handle for zero-copy operations */ - UCT_MD_FLAG_NEED_RKEY = UCS_BIT(3), /**< The transport needs a valid - remote memory key for remote memory - operations */ - UCT_MD_FLAG_ADVISE = UCS_BIT(4), /**< MD supports memory advice */ - UCT_MD_FLAG_FIXED = UCS_BIT(5), /**< MD supports memory allocation with - fixed address */ - UCT_MD_FLAG_RKEY_PTR = UCS_BIT(6), /**< MD supports direct access to - remote memory via a pointer that - is returned by @ref uct_rkey_ptr */ - UCT_MD_FLAG_SOCKADDR = UCS_BIT(7) /**< MD support for client-server - connection establishment via - sockaddr */ + UCT_MD_FLAG_ALLOC = UCS_BIT(0), /**< MD supports memory allocation */ + UCT_MD_FLAG_REG = UCS_BIT(1), /**< MD supports memory registration */ + UCT_MD_FLAG_NEED_MEMH = UCS_BIT(2), /**< The transport needs a valid local + memory handle for zero-copy operations */ + UCT_MD_FLAG_NEED_RKEY = UCS_BIT(3), /**< The transport needs a valid + remote memory key for remote memory + operations */ + UCT_MD_FLAG_ADVISE = UCS_BIT(4), /**< MD supports memory advice */ + UCT_MD_FLAG_FIXED = UCS_BIT(5), /**< MD supports memory allocation with + fixed address */ + UCT_MD_FLAG_RKEY_PTR = UCS_BIT(6), /**< MD supports direct access to + remote memory via a pointer that + is returned by @ref uct_rkey_ptr */ + UCT_MD_FLAG_SOCKADDR = UCS_BIT(7), /**< MD support for client-server + connection establishment via + sockaddr */ + UCT_MD_FLAG_DEVICE_ALLOC = UCS_BIT(8) /**< MD supports Device Memory allocation */ }; /* @@ -567,6 +568,9 @@ enum uct_md_mem_flags { UCT_MD_MEM_ACCESS_REMOTE_GET = UCS_BIT(6), /**< enable remote get access */ UCT_MD_MEM_ACCESS_REMOTE_ATOMIC = UCS_BIT(7), /**< enable remote atomic access */ + /* device specific flags */ + UCT_MD_MEM_FLAG_ON_DEVICE = UCS_BIT(8), /**< allocate memory on device */ + /** enable local and remote access for all operations */ UCT_MD_MEM_ACCESS_ALL = (UCT_MD_MEM_ACCESS_REMOTE_PUT| UCT_MD_MEM_ACCESS_REMOTE_GET| diff --git a/src/uct/base/uct_mem.c b/src/uct/base/uct_mem.c index cf8450402a7..292e5549e70 100644 --- a/src/uct/base/uct_mem.c +++ b/src/uct/base/uct_mem.c @@ -50,6 +50,31 @@ static inline int uct_mem_get_mmap_flags(unsigned uct_mmap_flags) return mm_flags; } +/* get number of devices which support flags */ +static int uct_mem_dm_flag_count(uct_md_h *mds, unsigned num_mds, int flags) +{ + ucs_status_t status; + unsigned md_index; + uct_md_attr_t md_attr; + uct_md_h md; + int cnt; + + for (cnt = 0, md_index = 0; md_index < num_mds; ++md_index) { + md = mds[md_index]; + status = uct_md_query(md, &md_attr); + if (status != UCS_OK) { + ucs_warn("Failed to query MD"); + continue; + } + + if (ucs_test_all_flags(md_attr.cap.flags, flags)) { + cnt++; + } + } + + return cnt; +} + ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, uct_alloc_method_t *methods, unsigned num_methods, uct_md_h *mds, unsigned num_mds, @@ -85,11 +110,31 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, return UCS_ERR_INVALID_PARAM; } + if ((flags & UCT_MD_MEM_FLAG_FIXED) && + (flags & UCT_MD_MEM_FLAG_ON_DEVICE)) { + ucs_debug("UCT_MD_MEM_FLAG_FIXED can't be used with UCT_MD_MEM_FLAG_ON_DEVICE"); + return UCS_ERR_INVALID_PARAM; + } + + if ((flags & UCT_MD_MEM_FLAG_NONBLOCK) && + (flags & UCT_MD_MEM_FLAG_ON_DEVICE)) { + ucs_debug("UCT_MD_MEM_FLAG_NONBLOCK can't be used with UCT_MD_MEM_FLAG_ON_DEVICE"); + return UCS_ERR_INVALID_PARAM; + } + for (method = methods; method < methods + num_methods; ++method) { ucs_trace("trying allocation method %s", uct_alloc_method_names[*method]); switch (*method) { case UCT_ALLOC_METHOD_MD: + if ((flags & UCT_MD_MEM_FLAG_ON_DEVICE) && + ((uct_mem_dm_flag_count(mds, num_mds, UCT_MD_FLAG_DEVICE_ALLOC) != 1) || + (uct_mem_dm_flag_count(mds, num_mds, UCT_MD_FLAG_REG) != 1))) { + /* DM allocation allowed on single device only, and no one else + * MD can register this memory */ + return UCS_ERR_NO_RESOURCE; + } + /* Allocate with one of the specified memory domains */ for (md_index = 0; md_index < num_mds; ++md_index) { md = mds[md_index]; @@ -99,15 +144,21 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, return status; } - /* Check if MD supports allocation */ - if (!(md_attr.cap.flags & UCT_MD_FLAG_ALLOC)) { + if ((flags & UCT_MD_MEM_FLAG_ON_DEVICE) && + !(md_attr.cap.flags & UCT_MD_FLAG_DEVICE_ALLOC)) { + /* DM requested, but not supported by MD */ + continue; + } + + if (!(flags & UCT_MD_MEM_FLAG_ON_DEVICE) && + !(md_attr.cap.flags & UCT_MD_FLAG_ALLOC)) { + /* DM is not requested, then ALLOC caps is required to allocate */ continue; } - /* Check if MD supports allocation with fixed address - * if it's requested */ if ((flags & UCT_MD_MEM_FLAG_FIXED) && !(md_attr.cap.flags & UCT_MD_FLAG_FIXED)) { + /* FIXED requested, but not supported by MD */ continue; } @@ -121,19 +172,22 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, status = uct_md_mem_alloc(md, &alloc_length, &address, flags, alloc_name, &memh); if (status != UCS_OK) { - ucs_error("failed to allocate %zu bytes using md %s for %s: %s", - alloc_length, md->component->name, - alloc_name, ucs_status_string(status)); + if (status != UCS_ERR_NO_RESOURCE) { + ucs_error("failed to allocate %zu bytes using md %s for %s: %s", + alloc_length, md->component->name, + alloc_name, ucs_status_string(status)); + } return status; } ucs_assert(memh != UCT_MEM_HANDLE_NULL); - mem->md = md; + mem->md = md; mem->mem_type = md_attr.cap.mem_type; - mem->memh = memh; + mem->memh = memh; goto allocated; } + break; case UCT_ALLOC_METHOD_THP: @@ -143,6 +197,10 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, break; } + if (flags & UCT_MD_MEM_FLAG_ON_DEVICE) { + break; + } + if (!ucs_is_thp_enabled()) { break; } @@ -181,6 +239,10 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, break; } + if (flags & UCT_MD_MEM_FLAG_ON_DEVICE) { + break; + } + alloc_length = min_length; address = ucs_memalign(UCS_SYS_CACHE_LINE_SIZE, alloc_length UCS_MEMTRACK_VAL); @@ -192,6 +254,10 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, break; case UCT_ALLOC_METHOD_MMAP: + if (flags & UCT_MD_MEM_FLAG_ON_DEVICE) { + break; + } + /* Request memory from operating system using mmap() */ alloc_length = min_length; address = addr; @@ -208,6 +274,10 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, break; case UCT_ALLOC_METHOD_HUGE: + if (flags & UCT_MD_MEM_FLAG_ON_DEVICE) { + break; + } + /* Allocate huge pages */ alloc_length = min_length; address = (flags & UCT_MD_MEM_FLAG_FIXED) ? addr : NULL; diff --git a/src/uct/ib/base/ib_md.c b/src/uct/ib/base/ib_md.c index 37e3940a12e..6bfc3dc74c3 100644 --- a/src/uct/ib/base/ib_md.c +++ b/src/uct/ib/base/ib_md.c @@ -223,6 +223,9 @@ static ucs_status_t uct_ib_md_query(uct_md_h uct_md, uct_md_attr_t *md_attr) UCT_MD_FLAG_NEED_MEMH | UCT_MD_FLAG_NEED_RKEY | UCT_MD_FLAG_ADVISE; +#if HAVE_IBV_EXP_DM + md_attr->cap.flags |= UCT_MD_FLAG_DEVICE_ALLOC; +#endif md_attr->cap.reg_mem_types = UCS_BIT(UCT_MD_MEM_TYPE_HOST); if (md->config.enable_gpudirect_rdma != UCS_NO) { @@ -484,7 +487,9 @@ ucs_status_t uct_ib_verbs_reg_atomic_key(uct_ib_md_t *md, { #if HAVE_EXP_UMR struct ibv_exp_mem_region *mem_reg = NULL; - struct ibv_mr *mr = memh->mr; + struct ibv_mr *mr = memh->mr; + int on_dm = memh->flags & UCT_IB_MEM_FLAG_DM; + uintptr_t base_addr = on_dm ? 0 : (uintptr_t)mr->addr; struct ibv_exp_send_wr wr, *bad_wr; struct ibv_exp_create_mr_in mrin; ucs_status_t status; @@ -557,7 +562,7 @@ ucs_status_t uct_ib_verbs_reg_atomic_key(uct_ib_md_t *md, } for (i = 0; i < list_size; i++) { - mem_reg[i].base_addr = (uintptr_t) mr->addr + i * reg_length; + mem_reg[i].base_addr = base_addr + i * reg_length; mem_reg[i].length = reg_length; mem_reg[i].mr = mr; } @@ -565,7 +570,7 @@ ucs_status_t uct_ib_verbs_reg_atomic_key(uct_ib_md_t *md, ucs_assert(list_size >= 1); mem_reg[list_size - 1].length = mr->length % reg_length; wr.ext_op.umr.mem_list.mem_reg_list = mem_reg; - wr.ext_op.umr.base_addr = (uint64_t) (uintptr_t) mr->addr + offset; + wr.ext_op.umr.base_addr = (uint64_t)base_addr + offset; wr.ext_op.umr.num_mrs = list_size; wr.ext_op.umr.modified_mr = umr; @@ -833,7 +838,45 @@ static void uct_ib_mem_init(uct_ib_mem_t *memh, unsigned uct_flags, if (uct_flags & UCT_MD_MEM_ACCESS_REMOTE_ATOMIC) { memh->flags |= UCT_IB_MEM_ACCESS_REMOTE_ATOMIC; } + + if (uct_flags & UCT_MD_MEM_FLAG_ON_DEVICE) { + memh->flags |= UCT_IB_MEM_FLAG_DM; + } +} + +#if HAVE_IBV_EXP_DM +/* TODO: here is code duplication with uct_rc_mlx5_iface_common_dm_tl_init to + * simplify backport it into v1.6 branch. deduplicate it after backport + * into v1.6 branch is complete */ +static ucs_status_t uct_ib_mem_alloc_dm(uct_ib_mem_t *memh, uct_ib_md_t *md, + size_t length, uint64_t exp_access) +{ + struct ibv_exp_alloc_dm_attr dm_attr = {.length = length}; + struct ibv_exp_reg_mr_in mr_in = {0}; + + memh->dm = UCS_PROFILE_CALL(ibv_exp_alloc_dm, md->dev.ibv_context, &dm_attr); + if (memh->dm == NULL) { + ucs_debug("failed to allocate memory on device: %s", uct_ib_device_name(&md->dev)); + return UCS_ERR_NO_RESOURCE; + } + + mr_in.pd = md->pd; + mr_in.length = length; + mr_in.exp_access = UCT_IB_MEM_ACCESS_FLAGS | exp_access; + mr_in.comp_mask = IBV_EXP_REG_MR_DM; + mr_in.dm = memh->dm; + + memh->mr = UCS_PROFILE_CALL(ibv_exp_reg_mr, &mr_in); + if (memh->mr == NULL) { + uct_ib_md_print_mem_reg_err_msg(UCS_LOG_LEVEL_DEBUG, mr_in.addr, mr_in.length, + mr_in.exp_access, "exp_"); + UCS_PROFILE_CALL(ibv_exp_free_dm, memh->dm); + return UCS_ERR_IO_ERROR; + } + + return UCS_OK; } +#endif static ucs_status_t uct_ib_mem_alloc(uct_md_h uct_md, size_t *length_p, void **address_p, unsigned flags, @@ -846,7 +889,17 @@ static ucs_status_t uct_ib_mem_alloc(uct_md_h uct_md, size_t *length_p, uct_ib_mem_t *memh; size_t length; - if (!md->config.enable_contig_pages) { +#if !HAVE_IBV_EXP_DM + if (flags & UCT_MD_MEM_FLAG_ON_DEVICE) { + return UCS_ERR_UNSUPPORTED; + } +#endif + + if (!(flags & UCT_MD_MEM_FLAG_ON_DEVICE) && + !md->config.enable_contig_pages) { + /* in case if ON_DEVICE flag is not active - then memory + * is allocated using contig_pages and contig pages + * should be enabled */ return UCS_ERR_UNSUPPORTED; } @@ -858,10 +911,24 @@ static ucs_status_t uct_ib_mem_alloc(uct_md_h uct_md, size_t *length_p, length = *length_p; exp_access = uct_ib_md_access_flags(md, flags, length) | - IBV_EXP_ACCESS_ALLOCATE_MR; - status = uct_ib_md_reg_mr(md, NULL, length, exp_access, 0, &memh->mr); - if (status != UCS_OK) { - goto err_free_memh; + !(flags & UCT_MD_MEM_FLAG_ON_DEVICE) ? + IBV_EXP_ACCESS_ALLOCATE_MR : 0; + +#if HAVE_IBV_EXP_DM + if (flags & UCT_MD_MEM_FLAG_ON_DEVICE) { + status = uct_ib_mem_alloc_dm(memh, md, length, exp_access); + if (status != UCS_OK) { + goto err_free_memh; + } + *address_p = ((uct_mlx5_dm_va_t*)memh->dm)->start_va; + } else +#endif + { + status = uct_ib_md_reg_mr(md, NULL, length, exp_access, 0, &memh->mr); + if (status != UCS_OK) { + goto err_free_memh; + } + *address_p = memh->mr->addr; } ucs_trace("allocated memory %p..%p on %s lkey 0x%x rkey 0x%x", @@ -878,7 +945,6 @@ static ucs_status_t uct_ib_mem_alloc(uct_md_h uct_md, size_t *length_p, UCS_STATS_UPDATE_COUNTER(md->stats, UCT_IB_MD_STAT_MEM_ALLOC, +1); ucs_memtrack_allocated(memh->mr->addr, memh->mr->length UCS_MEMTRACK_VAL); - *address_p = memh->mr->addr; *length_p = memh->mr->length; *memh_p = memh; return UCS_OK; @@ -915,6 +981,14 @@ static ucs_status_t uct_ib_mem_free(uct_md_h uct_md, uct_mem_h memh) return status; } + if (ib_memh->flags & UCT_IB_MEM_FLAG_DM) { +#if HAVE_IBV_EXP_DM + UCS_PROFILE_CALL(ibv_exp_free_dm, ib_memh->dm); +#else + ucs_assert_always(0); +#endif + } + uct_ib_memh_free(ib_memh); return UCS_OK; } diff --git a/src/uct/ib/base/ib_md.h b/src/uct/ib/base/ib_md.h index 245ce94b756..751b86f2b28 100644 --- a/src/uct/ib/base/ib_md.h +++ b/src/uct/ib/base/ib_md.h @@ -36,9 +36,11 @@ enum { demand paging enabled */ UCT_IB_MEM_FLAG_ATOMIC_MR = UCS_BIT(1), /**< The memory region has UMR for the atomic access */ - UCT_IB_MEM_ACCESS_REMOTE_ATOMIC = UCS_BIT(2) /**< An atomic access was + UCT_IB_MEM_ACCESS_REMOTE_ATOMIC = UCS_BIT(2), /**< An atomic access was requested for the memory region */ + UCT_IB_MEM_FLAG_DM = UCS_BIT(3) /**< The memory region is located + on device memory */ }; @@ -65,6 +67,16 @@ typedef struct uct_ib_md_ext_config { } uct_ib_md_ext_config_t; +#if HAVE_IBV_EXP_DM +/* uct_mlx5_dm_va is used to get pointer to DM mapped into process address space */ +typedef struct uct_mlx5_dm_va { + struct ibv_exp_dm ibv_dm; + size_t length; + uint64_t *start_va; +} uct_mlx5_dm_va_t; +#endif + + typedef struct uct_ib_mem { uint32_t lkey; uint32_t atomic_rkey; @@ -73,6 +85,7 @@ typedef struct uct_ib_mem { #if HAVE_EXP_UMR struct ibv_mr *atomic_mr; #endif + struct ibv_exp_dm *dm; } uct_ib_mem_t; struct uct_ib_md; diff --git a/src/uct/ib/mlx5/ib_mlx5_dv.h b/src/uct/ib/mlx5/ib_mlx5_dv.h index 2d23f35c0d3..bf051bc79a8 100644 --- a/src/uct/ib/mlx5/ib_mlx5_dv.h +++ b/src/uct/ib/mlx5/ib_mlx5_dv.h @@ -14,6 +14,8 @@ #include #include +#include + typedef struct { struct mlx5dv_obj dv; #if HAVE_IBV_EXP_DM @@ -95,12 +97,6 @@ ibv_reg_dm_mr(struct ibv_pd *pd, struct ibv_dm *dm, return ibv_exp_reg_mr(&mr_in); } -typedef struct uct_mlx5_dm_va { - struct ibv_dm ibv_dm; - size_t length; - uint64_t *start_va; -} uct_mlx5_dm_va_t; - static ucs_status_t UCS_F_MAYBE_UNUSED uct_ib_mlx5_get_dm_info(struct ibv_exp_dm *dm, struct mlx5dv_dm *dm_info) { diff --git a/test/gtest/ucp/test_ucp_mmap.cc b/test/gtest/ucp/test_ucp_mmap.cc index 965634b3ddc..1484a14f950 100644 --- a/test/gtest/ucp/test_ucp_mmap.cc +++ b/test/gtest/ucp/test_ucp_mmap.cc @@ -176,6 +176,88 @@ UCS_TEST_P(test_ucp_mmap, alloc) { } } +UCS_TEST_P(test_ucp_mmap, dm_alloc, "SHM_DEVICES=\"\"") { + int num_mds = 0; + ucp_mem_attr_t attr; + ucs_status_t status; + int i; + void *rkey_buffer; + size_t rkey_size; + ucp_rkey_h rkey; + void *buffer; + + if (!has_transport("dc_x") && !has_transport("rc_x") && + !has_transport("rc")) { + UCS_TEST_SKIP_R("not supported"); + } + + ucp_context_h context = sender().ucph(); + for (i = 0; i < context->num_mds; i++) { + if (context->tl_mds[i].attr.cap.flags & UCT_MD_FLAG_DEVICE_ALLOC) { + num_mds++; + if (num_mds > 1) { + UCS_TEST_SKIP_R("2+ MEMIC devices are not supported"); + } + } + } + + sender().connect(&sender(), get_ep_params()); + + for (int i = 1; i < 300; ++i) { + size_t size = i * 100; + + ucp_mem_h memh; + ucp_mem_map_params_t params; + + params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | + UCP_MEM_MAP_PARAM_FIELD_LENGTH | + UCP_MEM_MAP_PARAM_FIELD_FLAGS; + params.address = NULL; + params.length = size; + params.flags = UCP_MEM_MAP_ALLOCATE | UCP_MEM_MAP_FIXED; + + status = ucp_mem_map(sender().ucph(), ¶ms, &memh); + if (status == UCS_ERR_NO_MEMORY) { + break; + } + + ASSERT_UCS_OK(status); + + attr.field_mask = UCP_MEM_ATTR_FIELD_ADDRESS | + UCP_MEM_ATTR_FIELD_LENGTH; + status = ucp_mem_query(memh, &attr); + ASSERT_UCS_OK(status); + + EXPECT_TRUE(attr.address != NULL); + EXPECT_TRUE(attr.length >= size); + + status = ucp_rkey_pack(context, memh, &rkey_buffer, &rkey_size); + ASSERT_UCS_OK(status); + + status = ucp_ep_rkey_unpack(sender().ep(), rkey_buffer, &rkey); + ASSERT_UCS_OK(status); + + ucp_rkey_buffer_release(rkey_buffer); + + buffer = malloc(size); + /* trying put operation */ + status = ucp_put_nbi(sender().ep(), buffer, size, 0, rkey); + ASSERT_UCS_OK_OR_INPROGRESS(status); + flush_worker(sender()); + + /* trying get operation */ + status = ucp_get_nbi(sender().ep(), buffer, size, 0, rkey); + ASSERT_UCS_OK_OR_INPROGRESS(status); + flush_worker(sender()); + + free(buffer); + ucp_rkey_destroy(rkey); + + status = ucp_mem_unmap(sender().ucph(), memh); + ASSERT_UCS_OK(status); + } +} + UCS_TEST_P(test_ucp_mmap, reg) { ucs_status_t status; diff --git a/test/gtest/uct/test_md.cc b/test/gtest/uct/test_md.cc index 2f009dfa7ff..a37e4177f70 100644 --- a/test/gtest/uct/test_md.cc +++ b/test/gtest/uct/test_md.cc @@ -271,6 +271,39 @@ UCS_TEST_P(test_md, alloc) { } } +UCS_TEST_P(test_md, alloc_dm) { + size_t size, orig_size; + ucs_status_t status; + void *address; + uct_mem_h memh; + + check_caps(UCT_MD_FLAG_DEVICE_ALLOC, "DM allocation"); + + for (unsigned i = 0; i < 300; ++i) { + size = orig_size = i * 100; + if (size == 0) { + continue; + } + + status = uct_md_mem_alloc(md(), &size, &address, + UCT_MD_MEM_ACCESS_ALL | UCT_MD_MEM_FLAG_ON_DEVICE, + "test DM", &memh); + EXPECT_GT(size, 0ul); + + EXPECT_TRUE((status == UCS_OK) || (status == UCS_ERR_NO_RESOURCE)); + if (status == UCS_ERR_NO_RESOURCE) { + continue; + } + + EXPECT_GE(size, orig_size); + EXPECT_TRUE(address != NULL); + EXPECT_TRUE(memh != UCT_MEM_HANDLE_NULL); + + memset(address, 0xBB, size); + uct_md_mem_free(md(), memh); + } +} + UCS_TEST_P(test_md, mem_type_owned) { uct_md_attr_t md_attr; diff --git a/test/gtest/uct/test_mem.cc b/test/gtest/uct/test_mem.cc index a4fb58f5c22..048b6330469 100644 --- a/test/gtest/uct/test_mem.cc +++ b/test/gtest/uct/test_mem.cc @@ -207,6 +207,63 @@ UCS_TEST_P(test_mem, mmap_fixed) { } } +UCS_TEST_P(test_mem, dm_alloc) { + uct_alloc_method_t methods[] = {UCT_ALLOC_METHOD_MD, GetParam(), UCT_ALLOC_METHOD_HEAP}; + int skipped = 1; + uct_allocated_memory mem; + uct_md_resource_desc_t *md_resources; + uct_md_attr_t md_attr; + unsigned i, num_md_resources; + ucs_status_t status; + uct_md_h pd; + uct_md_config_t *md_config; + size_t j; + size_t length; + + status = uct_query_md_resources(&md_resources, &num_md_resources); + ASSERT_UCS_OK(status); + + for (i = 0; i < num_md_resources; ++i) { + status = uct_md_config_read(md_resources[i].md_name, NULL, NULL, &md_config); + ASSERT_UCS_OK(status); + + status = uct_md_open(md_resources[i].md_name, md_config, &pd); + uct_config_release(md_config); + ASSERT_UCS_OK(status); + + status = uct_md_query(pd, &md_attr); + ASSERT_UCS_OK(status); + + if (md_attr.cap.flags & UCT_MD_FLAG_DEVICE_ALLOC) { + skipped = 0; + + for (j = 1; j < 300; j++) { + length = j * 100; + status = uct_mem_alloc(NULL, length, + UCT_MD_MEM_ACCESS_ALL | UCT_MD_MEM_FLAG_ON_DEVICE, + methods, 3, &pd, 1, "test", &mem); + if (status == UCS_ERR_NO_RESOURCE) { + break; + } + ASSERT_UCS_OK(status); + + check_mem(mem, length); + + uct_mem_free(&mem); + } + } + + uct_md_close(pd); + } + + uct_release_md_resource_list(md_resources); + + if (skipped) { + /* mark test as skipped due to no device memory MD are processed */ + UCS_TEST_SKIP_R("not supported"); + } +} + INSTANTIATE_TEST_CASE_P(alloc_methods, test_mem, ::testing::Values(UCT_ALLOC_METHOD_THP, UCT_ALLOC_METHOD_HEAP, From fef892e82126b399989edeee6821fc6491ea5c72 Mon Sep 17 00:00:00 2001 From: Sergey Oblomov Date: Sun, 28 Apr 2019 13:56:02 +0300 Subject: [PATCH 2/2] IB/DM: removed external MEMIC flags --- src/ucp/core/ucp_mm.c | 8 +------- src/uct/api/uct.h | 3 --- src/uct/base/uct_mem.c | 33 +++++++++++++-------------------- src/uct/ib/base/ib_md.c | 20 ++++++++++---------- test/gtest/ucp/test_ucp_mmap.cc | 2 +- test/gtest/uct/test_md.cc | 12 +++++++----- test/gtest/uct/test_mem.cc | 2 +- 7 files changed, 33 insertions(+), 47 deletions(-) diff --git a/src/ucp/core/ucp_mm.c b/src/ucp/core/ucp_mm.c index d20df0ed9df..1ea72c3b157 100644 --- a/src/ucp/core/ucp_mm.c +++ b/src/ucp/core/ucp_mm.c @@ -219,13 +219,7 @@ ucp_mem_map_params2uct_flags(ucp_mem_map_params_t *params) } if (params->flags & UCP_MEM_MAP_FIXED) { - if ((params->field_mask & UCP_MEM_MAP_PARAM_FIELD_ADDRESS) && - (params->address == NULL)) { - /* special hint: allocate memory on device */ - flags |= UCT_MD_MEM_FLAG_ON_DEVICE; - } else { - flags |= UCT_MD_MEM_FLAG_FIXED; - } + flags |= UCT_MD_MEM_FLAG_FIXED; } } diff --git a/src/uct/api/uct.h b/src/uct/api/uct.h index f5dd740867e..09638ec4553 100644 --- a/src/uct/api/uct.h +++ b/src/uct/api/uct.h @@ -568,9 +568,6 @@ enum uct_md_mem_flags { UCT_MD_MEM_ACCESS_REMOTE_GET = UCS_BIT(6), /**< enable remote get access */ UCT_MD_MEM_ACCESS_REMOTE_ATOMIC = UCS_BIT(7), /**< enable remote atomic access */ - /* device specific flags */ - UCT_MD_MEM_FLAG_ON_DEVICE = UCS_BIT(8), /**< allocate memory on device */ - /** enable local and remote access for all operations */ UCT_MD_MEM_ACCESS_ALL = (UCT_MD_MEM_ACCESS_REMOTE_PUT| UCT_MD_MEM_ACCESS_REMOTE_GET| diff --git a/src/uct/base/uct_mem.c b/src/uct/base/uct_mem.c index 292e5549e70..8766b9dc66e 100644 --- a/src/uct/base/uct_mem.c +++ b/src/uct/base/uct_mem.c @@ -80,6 +80,7 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, uct_md_h *mds, unsigned num_mds, const char *alloc_name, uct_allocated_memory_t *mem) { + int on_dm = !addr && (flags & UCT_MD_MEM_FLAG_FIXED); uct_alloc_method_t *method; uct_md_attr_t md_attr; ucs_status_t status; @@ -105,20 +106,14 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, } if ((flags & UCT_MD_MEM_FLAG_FIXED) && - (!addr || ((uintptr_t)addr % ucs_get_page_size()))) { + ((uintptr_t)addr % ucs_get_page_size())) { ucs_debug("UCT_MD_MEM_FLAG_FIXED requires valid page size aligned address"); return UCS_ERR_INVALID_PARAM; } - if ((flags & UCT_MD_MEM_FLAG_FIXED) && - (flags & UCT_MD_MEM_FLAG_ON_DEVICE)) { - ucs_debug("UCT_MD_MEM_FLAG_FIXED can't be used with UCT_MD_MEM_FLAG_ON_DEVICE"); - return UCS_ERR_INVALID_PARAM; - } - - if ((flags & UCT_MD_MEM_FLAG_NONBLOCK) && - (flags & UCT_MD_MEM_FLAG_ON_DEVICE)) { - ucs_debug("UCT_MD_MEM_FLAG_NONBLOCK can't be used with UCT_MD_MEM_FLAG_ON_DEVICE"); + if ((flags & UCT_MD_MEM_FLAG_NONBLOCK) && on_dm) { + ucs_debug("UCT_MD_MEM_FLAG_NONBLOCK can't be used with " + "UCT_MD_MEM_FLAG_FIXED and NULL address"); return UCS_ERR_INVALID_PARAM; } @@ -127,7 +122,7 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, switch (*method) { case UCT_ALLOC_METHOD_MD: - if ((flags & UCT_MD_MEM_FLAG_ON_DEVICE) && + if (on_dm && ((uct_mem_dm_flag_count(mds, num_mds, UCT_MD_FLAG_DEVICE_ALLOC) != 1) || (uct_mem_dm_flag_count(mds, num_mds, UCT_MD_FLAG_REG) != 1))) { /* DM allocation allowed on single device only, and no one else @@ -144,19 +139,17 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, return status; } - if ((flags & UCT_MD_MEM_FLAG_ON_DEVICE) && - !(md_attr.cap.flags & UCT_MD_FLAG_DEVICE_ALLOC)) { + if (on_dm && !(md_attr.cap.flags & UCT_MD_FLAG_DEVICE_ALLOC)) { /* DM requested, but not supported by MD */ continue; } - if (!(flags & UCT_MD_MEM_FLAG_ON_DEVICE) && - !(md_attr.cap.flags & UCT_MD_FLAG_ALLOC)) { + if (!on_dm && !(md_attr.cap.flags & UCT_MD_FLAG_ALLOC)) { /* DM is not requested, then ALLOC caps is required to allocate */ continue; } - if ((flags & UCT_MD_MEM_FLAG_FIXED) && + if (!on_dm && (flags & UCT_MD_MEM_FLAG_FIXED) && !(md_attr.cap.flags & UCT_MD_FLAG_FIXED)) { /* FIXED requested, but not supported by MD */ continue; @@ -197,7 +190,7 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, break; } - if (flags & UCT_MD_MEM_FLAG_ON_DEVICE) { + if (on_dm) { break; } @@ -239,7 +232,7 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, break; } - if (flags & UCT_MD_MEM_FLAG_ON_DEVICE) { + if (on_dm) { break; } @@ -254,7 +247,7 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, break; case UCT_ALLOC_METHOD_MMAP: - if (flags & UCT_MD_MEM_FLAG_ON_DEVICE) { + if (on_dm) { break; } @@ -274,7 +267,7 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, break; case UCT_ALLOC_METHOD_HUGE: - if (flags & UCT_MD_MEM_FLAG_ON_DEVICE) { + if (on_dm) { break; } diff --git a/src/uct/ib/base/ib_md.c b/src/uct/ib/base/ib_md.c index 6bfc3dc74c3..29f424c5cdc 100644 --- a/src/uct/ib/base/ib_md.c +++ b/src/uct/ib/base/ib_md.c @@ -838,10 +838,6 @@ static void uct_ib_mem_init(uct_ib_mem_t *memh, unsigned uct_flags, if (uct_flags & UCT_MD_MEM_ACCESS_REMOTE_ATOMIC) { memh->flags |= UCT_IB_MEM_ACCESS_REMOTE_ATOMIC; } - - if (uct_flags & UCT_MD_MEM_FLAG_ON_DEVICE) { - memh->flags |= UCT_IB_MEM_FLAG_DM; - } } #if HAVE_IBV_EXP_DM @@ -884,19 +880,19 @@ static ucs_status_t uct_ib_mem_alloc(uct_md_h uct_md, size_t *length_p, { #if HAVE_DECL_IBV_EXP_ACCESS_ALLOCATE_MR uct_ib_md_t *md = ucs_derived_of(uct_md, uct_ib_md_t); + int on_dm = !(*address_p) && (flags & UCT_MD_MEM_FLAG_FIXED); ucs_status_t status; uint64_t exp_access; uct_ib_mem_t *memh; size_t length; #if !HAVE_IBV_EXP_DM - if (flags & UCT_MD_MEM_FLAG_ON_DEVICE) { + if (on_dm) { return UCS_ERR_UNSUPPORTED; } #endif - if (!(flags & UCT_MD_MEM_FLAG_ON_DEVICE) && - !md->config.enable_contig_pages) { + if (!on_dm && !md->config.enable_contig_pages) { /* in case if ON_DEVICE flag is not active - then memory * is allocated using contig_pages and contig pages * should be enabled */ @@ -911,11 +907,10 @@ static ucs_status_t uct_ib_mem_alloc(uct_md_h uct_md, size_t *length_p, length = *length_p; exp_access = uct_ib_md_access_flags(md, flags, length) | - !(flags & UCT_MD_MEM_FLAG_ON_DEVICE) ? - IBV_EXP_ACCESS_ALLOCATE_MR : 0; + (!on_dm ? IBV_EXP_ACCESS_ALLOCATE_MR : 0); #if HAVE_IBV_EXP_DM - if (flags & UCT_MD_MEM_FLAG_ON_DEVICE) { + if (on_dm) { status = uct_ib_mem_alloc_dm(memh, md, length, exp_access); if (status != UCS_OK) { goto err_free_memh; @@ -936,6 +931,11 @@ static ucs_status_t uct_ib_mem_alloc(uct_md_h uct_md, size_t *length_p, memh->mr->lkey, memh->mr->rkey); uct_ib_mem_init(memh, flags, exp_access); + + if (on_dm) { + memh->flags |= UCT_IB_MEM_FLAG_DM; + } + uct_ib_mem_set_numa_policy(md, memh); if (md->config.odp.prefetch) { diff --git a/test/gtest/ucp/test_ucp_mmap.cc b/test/gtest/ucp/test_ucp_mmap.cc index 1484a14f950..39134c31ecb 100644 --- a/test/gtest/ucp/test_ucp_mmap.cc +++ b/test/gtest/ucp/test_ucp_mmap.cc @@ -176,7 +176,7 @@ UCS_TEST_P(test_ucp_mmap, alloc) { } } -UCS_TEST_P(test_ucp_mmap, dm_alloc, "SHM_DEVICES=\"\"") { +UCS_TEST_P(test_ucp_mmap, dm_alloc) { int num_mds = 0; ucp_mem_attr_t attr; ucs_status_t status; diff --git a/test/gtest/uct/test_md.cc b/test/gtest/uct/test_md.cc index a37e4177f70..5552508f6e0 100644 --- a/test/gtest/uct/test_md.cc +++ b/test/gtest/uct/test_md.cc @@ -272,9 +272,9 @@ UCS_TEST_P(test_md, alloc) { } UCS_TEST_P(test_md, alloc_dm) { + void *address; size_t size, orig_size; ucs_status_t status; - void *address; uct_mem_h memh; check_caps(UCT_MD_FLAG_DEVICE_ALLOC, "DM allocation"); @@ -285,16 +285,18 @@ UCS_TEST_P(test_md, alloc_dm) { continue; } + address = NULL; + status = uct_md_mem_alloc(md(), &size, &address, - UCT_MD_MEM_ACCESS_ALL | UCT_MD_MEM_FLAG_ON_DEVICE, + UCT_MD_MEM_ACCESS_ALL | UCT_MD_MEM_FLAG_FIXED, "test DM", &memh); - EXPECT_GT(size, 0ul); - - EXPECT_TRUE((status == UCS_OK) || (status == UCS_ERR_NO_RESOURCE)); if (status == UCS_ERR_NO_RESOURCE) { continue; } + ASSERT_UCS_OK(status); + EXPECT_GT(size, 0ul); + EXPECT_GE(size, orig_size); EXPECT_TRUE(address != NULL); EXPECT_TRUE(memh != UCT_MEM_HANDLE_NULL); diff --git a/test/gtest/uct/test_mem.cc b/test/gtest/uct/test_mem.cc index 048b6330469..c48421ed19a 100644 --- a/test/gtest/uct/test_mem.cc +++ b/test/gtest/uct/test_mem.cc @@ -240,7 +240,7 @@ UCS_TEST_P(test_mem, dm_alloc) { for (j = 1; j < 300; j++) { length = j * 100; status = uct_mem_alloc(NULL, length, - UCT_MD_MEM_ACCESS_ALL | UCT_MD_MEM_FLAG_ON_DEVICE, + UCT_MD_MEM_ACCESS_ALL | UCT_MD_MEM_FLAG_FIXED, methods, 3, &pd, 1, "test", &mem); if (status == UCS_ERR_NO_RESOURCE) { break;