diff --git a/config/common_base b/config/common_base index 59bdde93ab..2725c3ca4d 100644 --- a/config/common_base +++ b/config/common_base @@ -206,6 +206,7 @@ CONFIG_RTE_LIBRTE_FM10K_INC_VECTOR=y # CONFIG_RTE_LIBRTE_MLX4_PMD=n CONFIG_RTE_LIBRTE_MLX4_DEBUG=n +CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS=n CONFIG_RTE_LIBRTE_MLX4_SGE_WR_N=4 CONFIG_RTE_LIBRTE_MLX4_MAX_INLINE=0 CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8 @@ -216,6 +217,7 @@ CONFIG_RTE_LIBRTE_MLX4_SOFT_COUNTERS=1 # CONFIG_RTE_LIBRTE_MLX5_PMD=n CONFIG_RTE_LIBRTE_MLX5_DEBUG=n +CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS=n CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8 # diff --git a/doc/guides/nics/mlx4.rst b/doc/guides/nics/mlx4.rst index 9a2e9736fe..ccfd4c3a3d 100644 --- a/doc/guides/nics/mlx4.rst +++ b/doc/guides/nics/mlx4.rst @@ -110,6 +110,24 @@ These options can be modified in the ``.config`` file. Toggle compilation of librte_pmd_mlx4 itself. +- ``CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS`` (default **n**) + + Build PMD with additional code to make it loadable without hard + dependencies on **libibverbs** nor **libmlx4**, which may not be installed + on the target system. + + In this mode, their presence is still required for it to run properly, + however their absence won't prevent a DPDK application from starting (with + ``CONFIG_RTE_BUILD_SHARED_LIB`` disabled) and they won't show up as + missing with ``ldd(1)``. + + It works by moving these dependencies to a purpose-built rdma-core "glue" + plug-in, which must either be installed in ``CONFIG_RTE_EAL_PMD_PATH`` if + set, or in a standard location for the dynamic linker (e.g. ``/lib``) if + left to the default empty string (``""``). + + This option has no performance impact. + - ``CONFIG_RTE_LIBRTE_MLX4_DEBUG`` (default **n**) Toggle debugging code and stricter compilation flags. Enabling this option @@ -144,6 +162,15 @@ These options can be modified in the ``.config`` file. Environment variables ~~~~~~~~~~~~~~~~~~~~~ +- ``MLX4_GLUE_PATH`` + + A list of directories in which to search for the rdma-core "glue" plug-in, + separated by colons or semi-colons. + + Only matters when compiled with ``CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS`` + enabled and most useful when ``CONFIG_RTE_EAL_PMD_PATH`` is also set, + since ``LD_LIBRARY_PATH`` has no effect in this case. + - ``MLX4_INLINE_RECV_SIZE`` A nonzero value enables inline receive for packets up to that size. May diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst index 09922a08f4..6293a9ce3a 100644 --- a/doc/guides/nics/mlx5.rst +++ b/doc/guides/nics/mlx5.rst @@ -111,6 +111,24 @@ These options can be modified in the ``.config`` file. Toggle compilation of librte_pmd_mlx5 itself. +- ``CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS`` (default **n**) + + Build PMD with additional code to make it loadable without hard + dependencies on **libibverbs** nor **libmlx5**, which may not be installed + on the target system. + + In this mode, their presence is still required for it to run properly, + however their absence won't prevent a DPDK application from starting (with + ``CONFIG_RTE_BUILD_SHARED_LIB`` disabled) and they won't show up as + missing with ``ldd(1)``. + + It works by moving these dependencies to a purpose-built rdma-core "glue" + plug-in, which must either be installed in ``CONFIG_RTE_EAL_PMD_PATH`` if + set, or in a standard location for the dynamic linker (e.g. ``/lib``) if + left to the default empty string (``""``). + + This option has no performance impact. + - ``CONFIG_RTE_LIBRTE_MLX5_DEBUG`` (default **n**) Toggle debugging code and stricter compilation flags. Enabling this option @@ -128,6 +146,15 @@ These options can be modified in the ``.config`` file. Environment variables ~~~~~~~~~~~~~~~~~~~~~ +- ``MLX5_GLUE_PATH`` + + A list of directories in which to search for the rdma-core "glue" plug-in, + separated by colons or semi-colons. + + Only matters when compiled with ``CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS`` + enabled and most useful when ``CONFIG_RTE_EAL_PMD_PATH`` is also set, + since ``LD_LIBRARY_PATH`` has no effect in this case. + - ``MLX5_PMD_ENABLE_PADDING`` Enables HW packet padding in PCI bus transactions. diff --git a/drivers/net/mlx4/Makefile b/drivers/net/mlx4/Makefile index 68c5902314..9348c07c16 100644 --- a/drivers/net/mlx4/Makefile +++ b/drivers/net/mlx4/Makefile @@ -33,9 +33,15 @@ include $(RTE_SDK)/mk/rte.vars.mk # Library name. LIB = librte_pmd_mlx4.a +LIB_GLUE = $(LIB_GLUE_BASE).$(LIB_GLUE_VERSION) +LIB_GLUE_BASE = librte_pmd_mlx4_glue.so +LIB_GLUE_VERSION = 17.02.1 # Sources. SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4.c +ifneq ($(CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS),y) +SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_glue.c +endif # Dependencies. DEPDIRS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += lib/librte_ether @@ -43,6 +49,10 @@ DEPDIRS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += lib/librte_mbuf DEPDIRS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += lib/librte_eal DEPDIRS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += lib/librte_mempool +ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS),y) +INSTALL-$(CONFIG_RTE_LIBRTE_MLX4_PMD)-lib += $(LIB_GLUE) +endif + # Basic CFLAGS. CFLAGS += -O3 CFLAGS += -std=gnu99 -Wall -Wextra @@ -52,7 +62,14 @@ CFLAGS += -D_BSD_SOURCE CFLAGS += -D_DEFAULT_SOURCE CFLAGS += -D_XOPEN_SOURCE=600 CFLAGS += $(WERROR_FLAGS) +ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS),y) +CFLAGS += -DMLX4_GLUE='"$(LIB_GLUE)"' +CFLAGS += -DMLX4_GLUE_VERSION='"$(LIB_GLUE_VERSION)"' +CFLAGS_mlx4_glue.o += -fPIC +LDLIBS += -ldl +else LDLIBS += -libverbs +endif # A few warnings cannot be avoided in external headers. CFLAGS += -Wno-error=cast-qual @@ -131,7 +148,24 @@ mlx4_autoconf.h: mlx4_autoconf.h.new mlx4.o: mlx4_autoconf.h +# Generate dependency plug-in for rdma-core when the PMD must not be linked +# directly, so that applications do not inherit this dependency. + +ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS),y) + +$(LIB): $(LIB_GLUE) + +$(LIB_GLUE): mlx4_glue.o + $Q $(LD) $(LDFLAGS) $(EXTRA_LDFLAGS) \ + -Wl,-h,$(LIB_GLUE) \ + -s -shared -o $@ $< -libverbs + +mlx4_glue.o: mlx4_autoconf.h + +endif + clean_mlx4: FORCE $Q rm -f -- mlx4_autoconf.h mlx4_autoconf.h.new + $Q rm -f -- mlx4_glue.o $(LIB_GLUE_BASE)* clean: clean_mlx4 diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c index 79efaaa3ce..4b2e1f54c3 100644 --- a/drivers/net/mlx4/mlx4.c +++ b/drivers/net/mlx4/mlx4.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -72,6 +73,7 @@ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic" #endif +#include #include #include #include @@ -95,6 +97,7 @@ /* PMD header. */ #include "mlx4.h" +#include "mlx4_glue.h" /* Runtime logging through RTE_LOG() is enabled when not in debugging mode. * Intermediate LOG_*() macros add the required end-of-line characters. */ @@ -1003,8 +1006,9 @@ txq_alloc_elts(struct txq *txq, unsigned int elts_n) goto error; } mr_linear = - ibv_reg_mr(txq->priv->pd, elts_linear, sizeof(*elts_linear), - IBV_ACCESS_LOCAL_WRITE); + mlx4_glue->reg_mr(txq->priv->pd, elts_linear, + sizeof(*elts_linear), + IBV_ACCESS_LOCAL_WRITE); if (mr_linear == NULL) { ERROR("%p: unable to configure MR, ibv_reg_mr() failed", (void *)txq); @@ -1034,7 +1038,7 @@ txq_alloc_elts(struct txq *txq, unsigned int elts_n) return 0; error: if (mr_linear != NULL) - claim_zero(ibv_dereg_mr(mr_linear)); + claim_zero(mlx4_glue->dereg_mr(mr_linear)); rte_free(elts_linear); rte_free(elts); @@ -1071,7 +1075,7 @@ txq_free_elts(struct txq *txq) txq->elts_linear = NULL; txq->mr_linear = NULL; if (mr_linear != NULL) - claim_zero(ibv_dereg_mr(mr_linear)); + claim_zero(mlx4_glue->dereg_mr(mr_linear)); rte_free(elts_linear); if (elts == NULL) @@ -1115,9 +1119,9 @@ txq_cleanup(struct txq *txq) params = (struct ibv_exp_release_intf_params){ .comp_mask = 0, }; - claim_zero(ibv_exp_release_intf(txq->priv->ctx, - txq->if_qp, - ¶ms)); + claim_zero(mlx4_glue->exp_release_intf(txq->priv->ctx, + txq->if_qp, + ¶ms)); } if (txq->if_cq != NULL) { assert(txq->priv != NULL); @@ -1126,14 +1130,14 @@ txq_cleanup(struct txq *txq) params = (struct ibv_exp_release_intf_params){ .comp_mask = 0, }; - claim_zero(ibv_exp_release_intf(txq->priv->ctx, - txq->if_cq, - ¶ms)); + claim_zero(mlx4_glue->exp_release_intf(txq->priv->ctx, + txq->if_cq, + ¶ms)); } if (txq->qp != NULL) - claim_zero(ibv_destroy_qp(txq->qp)); + claim_zero(mlx4_glue->destroy_qp(txq->qp)); if (txq->cq != NULL) - claim_zero(ibv_destroy_cq(txq->cq)); + claim_zero(mlx4_glue->destroy_cq(txq->cq)); if (txq->rd != NULL) { struct ibv_exp_destroy_res_domain_attr attr = { .comp_mask = 0, @@ -1141,15 +1145,15 @@ txq_cleanup(struct txq *txq) assert(txq->priv != NULL); assert(txq->priv->ctx != NULL); - claim_zero(ibv_exp_destroy_res_domain(txq->priv->ctx, - txq->rd, - &attr)); + claim_zero(mlx4_glue->exp_destroy_res_domain(txq->priv->ctx, + txq->rd, + &attr)); } for (i = 0; (i != elemof(txq->mp2mr)); ++i) { if (txq->mp2mr[i].mp == NULL) break; assert(txq->mp2mr[i].mr != NULL); - claim_zero(ibv_dereg_mr(txq->mp2mr[i].mr)); + claim_zero(mlx4_glue->dereg_mr(txq->mp2mr[i].mr)); } memset(txq, 0, sizeof(*txq)); } @@ -1313,10 +1317,10 @@ mlx4_mp2mr(struct ibv_pd *pd, struct rte_mempool *mp) DEBUG("mempool %p using start=%p end=%p size=%zu for MR", (void *)mp, (void *)start, (void *)end, (size_t)(end - start)); - return ibv_reg_mr(pd, - (void *)start, - end - start, - IBV_ACCESS_LOCAL_WRITE); + return mlx4_glue->reg_mr(pd, + (void *)start, + end - start, + IBV_ACCESS_LOCAL_WRITE); } /** @@ -1381,7 +1385,7 @@ txq_mp2mr(struct txq *txq, struct rte_mempool *mp) DEBUG("%p: MR <-> MP table full, dropping oldest entry.", (void *)txq); --i; - claim_zero(ibv_dereg_mr(txq->mp2mr[0].mr)); + claim_zero(mlx4_glue->dereg_mr(txq->mp2mr[0].mr)); memmove(&txq->mp2mr[0], &txq->mp2mr[1], (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0]))); } @@ -1874,7 +1878,7 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, .thread_model = IBV_EXP_THREAD_SINGLE, .msg_model = IBV_EXP_MSG_HIGH_BW, }; - tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); + tmpl.rd = mlx4_glue->exp_create_res_domain(priv->ctx, &attr.rd); if (tmpl.rd == NULL) { ret = ENOMEM; ERROR("%p: RD creation failure: %s", @@ -1885,7 +1889,8 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, .res_domain = tmpl.rd, }; - tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, &attr.cq); + tmpl.cq = mlx4_glue->exp_create_cq(priv->ctx, desc, NULL, NULL, 0, + &attr.cq); if (tmpl.cq == NULL) { ret = ENOMEM; ERROR("%p: CQ creation failure: %s", @@ -1924,7 +1929,7 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | IBV_EXP_QP_INIT_ATTR_RES_DOMAIN), }; - tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init); + tmpl.qp = mlx4_glue->exp_create_qp(priv->ctx, &attr.init); if (tmpl.qp == NULL) { ret = (errno ? errno : EINVAL); ERROR("%p: QP creation failure: %s", @@ -1941,8 +1946,8 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, /* Primary port number. */ .port_num = priv->port }; - ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, - (IBV_EXP_QP_STATE | IBV_EXP_QP_PORT)); + ret = mlx4_glue->exp_modify_qp(tmpl.qp, &attr.mod, + (IBV_EXP_QP_STATE | IBV_EXP_QP_PORT)); if (ret) { ERROR("%p: QP state to IBV_QPS_INIT failed: %s", (void *)dev, strerror(ret)); @@ -1957,14 +1962,14 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, attr.mod = (struct ibv_exp_qp_attr){ .qp_state = IBV_QPS_RTR }; - ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE); + ret = mlx4_glue->exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE); if (ret) { ERROR("%p: QP state to IBV_QPS_RTR failed: %s", (void *)dev, strerror(ret)); goto error; } attr.mod.qp_state = IBV_QPS_RTS; - ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE); + ret = mlx4_glue->exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE); if (ret) { ERROR("%p: QP state to IBV_QPS_RTS failed: %s", (void *)dev, strerror(ret)); @@ -1975,7 +1980,8 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, .intf = IBV_EXP_INTF_CQ, .obj = tmpl.cq, }; - tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); + tmpl.if_cq = mlx4_glue->exp_query_intf(priv->ctx, &attr.params, + &status); if (tmpl.if_cq == NULL) { ERROR("%p: CQ interface family query failed with status %d", (void *)dev, status); @@ -1993,7 +1999,8 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, 0), #endif }; - tmpl.if_qp = ibv_exp_query_intf(priv->ctx, &attr.params, &status); + tmpl.if_qp = mlx4_glue->exp_query_intf(priv->ctx, &attr.params, + &status); if (tmpl.if_qp == NULL) { ERROR("%p: QP interface family query failed with status %d", (void *)dev, status); @@ -2434,7 +2441,8 @@ rxq_del_flow(struct rxq *rxq, unsigned int mac_index, unsigned int vlan_index) (void *)rxq, (*mac)[0], (*mac)[1], (*mac)[2], (*mac)[3], (*mac)[4], (*mac)[5], mac_index, priv->vlan_filter[vlan_index].id); - claim_zero(ibv_destroy_flow(rxq->mac_flow[mac_index][vlan_index])); + claim_zero(mlx4_glue->destroy_flow + (rxq->mac_flow[mac_index][vlan_index])); rxq->mac_flow[mac_index][vlan_index] = NULL; } @@ -2556,7 +2564,7 @@ rxq_add_flow(struct rxq *rxq, unsigned int mac_index, unsigned int vlan_index) ((vlan_index != -1u) ? priv->vlan_filter[vlan_index].id : -1u)); /* Create related flow. */ errno = 0; - flow = ibv_create_flow(rxq->qp, attr); + flow = mlx4_glue->create_flow(rxq->qp, attr); if (flow == NULL) { /* It's not clear whether errno is always set in this case. */ ERROR("%p: flow configuration failed, errno=%d: %s", @@ -2786,7 +2794,7 @@ rxq_allmulticast_enable(struct rxq *rxq) if (rxq->allmulti_flow != NULL) return EBUSY; errno = 0; - flow = ibv_create_flow(rxq->qp, &attr); + flow = mlx4_glue->create_flow(rxq->qp, &attr); if (flow == NULL) { /* It's not clear whether errno is always set in this case. */ ERROR("%p: flow configuration failed, errno=%d: %s", @@ -2813,7 +2821,7 @@ rxq_allmulticast_disable(struct rxq *rxq) DEBUG("%p: disabling allmulticast mode", (void *)rxq); if (rxq->allmulti_flow == NULL) return; - claim_zero(ibv_destroy_flow(rxq->allmulti_flow)); + claim_zero(mlx4_glue->destroy_flow(rxq->allmulti_flow)); rxq->allmulti_flow = NULL; DEBUG("%p: allmulticast mode disabled", (void *)rxq); } @@ -2844,7 +2852,7 @@ rxq_promiscuous_enable(struct rxq *rxq) if (rxq->promisc_flow != NULL) return EBUSY; errno = 0; - flow = ibv_create_flow(rxq->qp, &attr); + flow = mlx4_glue->create_flow(rxq->qp, &attr); if (flow == NULL) { /* It's not clear whether errno is always set in this case. */ ERROR("%p: flow configuration failed, errno=%d: %s", @@ -2873,7 +2881,7 @@ rxq_promiscuous_disable(struct rxq *rxq) DEBUG("%p: disabling promiscuous mode", (void *)rxq); if (rxq->promisc_flow == NULL) return; - claim_zero(ibv_destroy_flow(rxq->promisc_flow)); + claim_zero(mlx4_glue->destroy_flow(rxq->promisc_flow)); rxq->promisc_flow = NULL; DEBUG("%p: promiscuous mode disabled", (void *)rxq); } @@ -2903,9 +2911,9 @@ rxq_cleanup(struct rxq *rxq) params = (struct ibv_exp_release_intf_params){ .comp_mask = 0, }; - claim_zero(ibv_exp_release_intf(rxq->priv->ctx, - rxq->if_qp, - ¶ms)); + claim_zero(mlx4_glue->exp_release_intf(rxq->priv->ctx, + rxq->if_qp, + ¶ms)); } if (rxq->if_cq != NULL) { assert(rxq->priv != NULL); @@ -2914,18 +2922,18 @@ rxq_cleanup(struct rxq *rxq) params = (struct ibv_exp_release_intf_params){ .comp_mask = 0, }; - claim_zero(ibv_exp_release_intf(rxq->priv->ctx, - rxq->if_cq, - ¶ms)); + claim_zero(mlx4_glue->exp_release_intf(rxq->priv->ctx, + rxq->if_cq, + ¶ms)); } if (rxq->qp != NULL) { rxq_promiscuous_disable(rxq); rxq_allmulticast_disable(rxq); rxq_mac_addrs_del(rxq); - claim_zero(ibv_destroy_qp(rxq->qp)); + claim_zero(mlx4_glue->destroy_qp(rxq->qp)); } if (rxq->cq != NULL) - claim_zero(ibv_destroy_cq(rxq->cq)); + claim_zero(mlx4_glue->destroy_cq(rxq->cq)); if (rxq->rd != NULL) { struct ibv_exp_destroy_res_domain_attr attr = { .comp_mask = 0, @@ -2933,12 +2941,12 @@ rxq_cleanup(struct rxq *rxq) assert(rxq->priv != NULL); assert(rxq->priv->ctx != NULL); - claim_zero(ibv_exp_destroy_res_domain(rxq->priv->ctx, - rxq->rd, - &attr)); + claim_zero(mlx4_glue->exp_destroy_res_domain(rxq->priv->ctx, + rxq->rd, + &attr)); } if (rxq->mr != NULL) - claim_zero(ibv_dereg_mr(rxq->mr)); + claim_zero(mlx4_glue->dereg_mr(rxq->mr)); memset(rxq, 0, sizeof(*rxq)); } @@ -3082,7 +3090,7 @@ mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) DEBUG("rxq=%p, poll_length() failed (ret=%d)", (void *)rxq, ret); /* ibv_poll_cq() must be used in case of failure. */ - wcs_n = ibv_poll_cq(rxq->cq, 1, &wc); + wcs_n = mlx4_glue->poll_cq(rxq->cq, 1, &wc); if (unlikely(wcs_n == 0)) break; if (unlikely(wcs_n < 0)) { @@ -3096,7 +3104,7 @@ mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) DEBUG("rxq=%p, wr_id=%" PRIu64 ": bad work" " completion status (%d): %s", (void *)rxq, wc.wr_id, wc.status, - ibv_wc_status_str(wc.status)); + mlx4_glue->wc_status_str(wc.status)); #ifdef MLX4_PMD_SOFT_COUNTERS /* Increment dropped packets counter. */ ++rxq->stats.idropped; @@ -3221,7 +3229,7 @@ mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) #ifdef DEBUG_RECV DEBUG("%p: reposting %d WRs", (void *)rxq, i); #endif - ret = ibv_post_recv(rxq->qp, head.next, &bad_wr); + ret = mlx4_glue->post_recv(rxq->qp, head.next, &bad_wr); if (unlikely(ret)) { /* Inability to repost WRs is fatal. */ DEBUG("%p: ibv_post_recv(): failed for WR %p: %s", @@ -3300,7 +3308,7 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) DEBUG("rxq=%p, poll_length() failed (ret=%d)", (void *)rxq, ret); /* ibv_poll_cq() must be used in case of failure. */ - wcs_n = ibv_poll_cq(rxq->cq, 1, &wc); + wcs_n = mlx4_glue->poll_cq(rxq->cq, 1, &wc); if (unlikely(wcs_n == 0)) break; if (unlikely(wcs_n < 0)) { @@ -3314,7 +3322,7 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) DEBUG("rxq=%p, wr_id=%" PRIu64 ": bad work" " completion status (%d): %s", (void *)rxq, wc.wr_id, wc.status, - ibv_wc_status_str(wc.status)); + mlx4_glue->wc_status_str(wc.status)); #ifdef MLX4_PMD_SOFT_COUNTERS /* Increment dropped packets counter. */ ++rxq->stats.idropped; @@ -3483,7 +3491,7 @@ rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc, attr.max_inl_recv = priv->inl_recv_size; attr.comp_mask |= IBV_EXP_QP_INIT_ATTR_INL_RECV; #endif - return ibv_exp_create_qp(priv->ctx, &attr); + return mlx4_glue->exp_create_qp(priv->ctx, &attr); } #ifdef RSS_SUPPORT @@ -3548,7 +3556,7 @@ rxq_setup_qp_rss(struct priv *priv, struct ibv_cq *cq, uint16_t desc, attr.qpg.qpg_parent = priv->rxq_parent.qp; DEBUG("initializing child RSS queue"); } - return ibv_exp_create_qp(priv->ctx, &attr); + return mlx4_glue->exp_create_qp(priv->ctx, &attr); } #endif /* RSS_SUPPORT */ @@ -3633,13 +3641,13 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) /* From now on, any failure will render the queue unusable. * Reinitialize QP. */ mod = (struct ibv_exp_qp_attr){ .qp_state = IBV_QPS_RESET }; - err = ibv_exp_modify_qp(tmpl.qp, &mod, IBV_EXP_QP_STATE); + err = mlx4_glue->exp_modify_qp(tmpl.qp, &mod, IBV_EXP_QP_STATE); if (err) { ERROR("%p: cannot reset QP: %s", (void *)dev, strerror(err)); assert(err > 0); return err; } - err = ibv_resize_cq(tmpl.cq, desc_n); + err = mlx4_glue->resize_cq(tmpl.cq, desc_n); if (err) { ERROR("%p: cannot resize CQ: %s", (void *)dev, strerror(err)); assert(err > 0); @@ -3651,12 +3659,12 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) /* Primary port number. */ .port_num = priv->port }; - err = ibv_exp_modify_qp(tmpl.qp, &mod, - (IBV_EXP_QP_STATE | + err = mlx4_glue->exp_modify_qp(tmpl.qp, &mod, + (IBV_EXP_QP_STATE | #ifdef RSS_SUPPORT - (parent ? IBV_EXP_QP_GROUP_RSS : 0) | + (parent ? IBV_EXP_QP_GROUP_RSS : 0) | #endif /* RSS_SUPPORT */ - IBV_EXP_QP_PORT)); + IBV_EXP_QP_PORT)); if (err) { ERROR("%p: QP state to IBV_QPS_INIT failed: %s", (void *)dev, strerror(err)); @@ -3731,11 +3739,11 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) rte_free(rxq->elts.sp); rxq->elts.sp = NULL; /* Post WRs. */ - err = ibv_post_recv(tmpl.qp, - (tmpl.sp ? - &(*tmpl.elts.sp)[0].wr : - &(*tmpl.elts.no_sp)[0].wr), - &bad_wr); + err = mlx4_glue->post_recv(tmpl.qp, + (tmpl.sp ? + &(*tmpl.elts.sp)[0].wr : + &(*tmpl.elts.no_sp)[0].wr), + &bad_wr); if (err) { ERROR("%p: ibv_post_recv() failed for WR %p: %s", (void *)dev, @@ -3746,7 +3754,7 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) mod = (struct ibv_exp_qp_attr){ .qp_state = IBV_QPS_RTR }; - err = ibv_exp_modify_qp(tmpl.qp, &mod, IBV_EXP_QP_STATE); + err = mlx4_glue->exp_modify_qp(tmpl.qp, &mod, IBV_EXP_QP_STATE); if (err) ERROR("%p: QP state to IBV_QPS_RTR failed: %s", (void *)dev, strerror(err)); @@ -3849,7 +3857,7 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, .thread_model = IBV_EXP_THREAD_SINGLE, .msg_model = IBV_EXP_MSG_HIGH_BW, }; - tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); + tmpl.rd = mlx4_glue->exp_create_res_domain(priv->ctx, &attr.rd); if (tmpl.rd == NULL) { ret = ENOMEM; ERROR("%p: RD creation failure: %s", @@ -3860,7 +3868,8 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, .res_domain = tmpl.rd, }; - tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, &attr.cq); + tmpl.cq = mlx4_glue->exp_create_cq(priv->ctx, desc, NULL, NULL, 0, + &attr.cq); if (tmpl.cq == NULL) { ret = ENOMEM; ERROR("%p: CQ creation failure: %s", @@ -3890,12 +3899,12 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, /* Primary port number. */ .port_num = priv->port }; - ret = ibv_exp_modify_qp(tmpl.qp, &mod, - (IBV_EXP_QP_STATE | + ret = mlx4_glue->exp_modify_qp(tmpl.qp, &mod, + (IBV_EXP_QP_STATE | #ifdef RSS_SUPPORT - (parent ? IBV_EXP_QP_GROUP_RSS : 0) | + (parent ? IBV_EXP_QP_GROUP_RSS : 0) | #endif /* RSS_SUPPORT */ - IBV_EXP_QP_PORT)); + IBV_EXP_QP_PORT)); if (ret) { ERROR("%p: QP state to IBV_QPS_INIT failed: %s", (void *)dev, strerror(ret)); @@ -3922,11 +3931,11 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, (void *)dev, strerror(ret)); goto error; } - ret = ibv_post_recv(tmpl.qp, - (tmpl.sp ? - &(*tmpl.elts.sp)[0].wr : - &(*tmpl.elts.no_sp)[0].wr), - &bad_wr); + ret = mlx4_glue->post_recv(tmpl.qp, + (tmpl.sp ? + &(*tmpl.elts.sp)[0].wr : + &(*tmpl.elts.no_sp)[0].wr), + &bad_wr); if (ret) { ERROR("%p: ibv_post_recv() failed for WR %p: %s", (void *)dev, @@ -3938,7 +3947,7 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, mod = (struct ibv_exp_qp_attr){ .qp_state = IBV_QPS_RTR }; - ret = ibv_exp_modify_qp(tmpl.qp, &mod, IBV_EXP_QP_STATE); + ret = mlx4_glue->exp_modify_qp(tmpl.qp, &mod, IBV_EXP_QP_STATE); if (ret) { ERROR("%p: QP state to IBV_QPS_RTR failed: %s", (void *)dev, strerror(ret)); @@ -3952,7 +3961,8 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, .intf = IBV_EXP_INTF_CQ, .obj = tmpl.cq, }; - tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); + tmpl.if_cq = mlx4_glue->exp_query_intf(priv->ctx, &attr.params, + &status); if (tmpl.if_cq == NULL) { ERROR("%p: CQ interface family query failed with status %d", (void *)dev, status); @@ -3963,7 +3973,8 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, .intf = IBV_EXP_INTF_QP_BURST, .obj = tmpl.qp, }; - tmpl.if_qp = ibv_exp_query_intf(priv->ctx, &attr.params, &status); + tmpl.if_qp = mlx4_glue->exp_query_intf(priv->ctx, &attr.params, + &status); if (tmpl.if_qp == NULL) { ERROR("%p: QP interface family query failed with status %d", (void *)dev, status); @@ -4319,8 +4330,8 @@ mlx4_dev_close(struct rte_eth_dev *dev) rxq_cleanup(&priv->rxq_parent); if (priv->pd != NULL) { assert(priv->ctx != NULL); - claim_zero(ibv_dealloc_pd(priv->pd)); - claim_zero(ibv_close_device(priv->ctx)); + claim_zero(mlx4_glue->dealloc_pd(priv->pd)); + claim_zero(mlx4_glue->close_device(priv->ctx)); } else assert(priv->ctx == NULL); priv_dev_interrupt_handler_uninstall(priv, dev); @@ -5383,7 +5394,7 @@ priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev) /* Read all message and acknowledge them. */ for (;;) { - if (ibv_get_async_event(priv->ctx, &event)) + if (mlx4_glue->get_async_event(priv->ctx, &event)) break; if (event.event_type == IBV_EVENT_PORT_ACTIVE || @@ -5392,7 +5403,7 @@ priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev) else DEBUG("event type %d on port %d not handled", event.event_type, event.element.port_num); - ibv_ack_async_event(&event); + mlx4_glue->ack_async_event(&event); } if (port_change ^ priv->pending_alarm) { @@ -5550,7 +5561,7 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) /* Save PCI address. */ mlx4_dev[idx].pci_addr = pci_dev->addr; - list = ibv_get_device_list(&i); + list = mlx4_glue->get_device_list(&i); if (list == NULL) { assert(errno); if (errno == ENOSYS) { @@ -5580,12 +5591,12 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) PCI_DEVICE_ID_MELLANOX_CONNECTX3VF); INFO("PCI information matches, using device \"%s\" (VF: %s)", list[i]->name, (vf ? "true" : "false")); - attr_ctx = ibv_open_device(list[i]); + attr_ctx = mlx4_glue->open_device(list[i]); err = errno; break; } if (attr_ctx == NULL) { - ibv_free_device_list(list); + mlx4_glue->free_device_list(list); switch (err) { case 0: WARN("cannot access device, is mlx4_ib loaded?"); @@ -5600,7 +5611,7 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) ibv_dev = list[i]; DEBUG("device opened"); - if (ibv_query_device(attr_ctx, &device_attr)) + if (mlx4_glue->query_device(attr_ctx, &device_attr)) goto error; INFO("%u port(s) detected", device_attr.phys_port_cnt); @@ -5626,12 +5637,12 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) DEBUG("using port %u (%08" PRIx32 ")", port, test); - ctx = ibv_open_device(ibv_dev); + ctx = mlx4_glue->open_device(ibv_dev); if (ctx == NULL) goto port_error; /* Check port status. */ - err = ibv_query_port(ctx, port, &port_attr); + err = mlx4_glue->query_port(ctx, port, &port_attr); if (err) { ERROR("port query failed: %s", strerror(err)); goto port_error; @@ -5645,11 +5656,11 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) if (port_attr.state != IBV_PORT_ACTIVE) DEBUG("port %d is not active: \"%s\" (%d)", - port, ibv_port_state_str(port_attr.state), + port, mlx4_glue->port_state_str(port_attr.state), port_attr.state); /* Allocate protection domain. */ - pd = ibv_alloc_pd(ctx); + pd = mlx4_glue->alloc_pd(ctx); if (pd == NULL) { ERROR("PD allocation failure"); err = ENOMEM; @@ -5674,7 +5685,7 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) priv->pd = pd; priv->mtu = ETHER_MTU; #ifdef HAVE_EXP_QUERY_DEVICE - if (ibv_exp_query_device(ctx, &exp_device_attr)) { + if (mlx4_glue->exp_query_device(ctx, &exp_device_attr)) { ERROR("ibv_exp_query_device() failed"); goto port_error; } @@ -5724,7 +5735,8 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) if (priv->inl_recv_size) { exp_device_attr.comp_mask = IBV_EXP_DEVICE_ATTR_INLINE_RECV_SZ; - if (ibv_exp_query_device(ctx, &exp_device_attr)) { + if (mlx4_glue->exp_query_device(ctx, + &exp_device_attr)) { INFO("Couldn't query device for inline-receive" " capabilities."); priv->inl_recv_size = 0; @@ -5785,7 +5797,7 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) char name[RTE_ETH_NAME_MAX_LEN]; snprintf(name, sizeof(name), "%s port %u", - ibv_get_device_name(ibv_dev), port); + mlx4_glue->get_device_name(ibv_dev), port); eth_dev = rte_eth_dev_allocate(name); } if (eth_dev == NULL) { @@ -5841,9 +5853,9 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) port_error: rte_free(priv); if (pd) - claim_zero(ibv_dealloc_pd(pd)); + claim_zero(mlx4_glue->dealloc_pd(pd)); if (ctx) - claim_zero(ibv_close_device(ctx)); + claim_zero(mlx4_glue->close_device(ctx)); if (eth_dev) rte_eth_dev_release_port(eth_dev); break; @@ -5864,9 +5876,9 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) error: if (attr_ctx) - claim_zero(ibv_close_device(attr_ctx)); + claim_zero(mlx4_glue->close_device(attr_ctx)); if (list) - ibv_free_device_list(list); + mlx4_glue->free_device_list(list); assert(err >= 0); return -err; } @@ -5901,6 +5913,88 @@ static struct eth_driver mlx4_driver = { .dev_private_size = sizeof(struct priv) }; +#ifdef RTE_LIBRTE_MLX4_DLOPEN_DEPS + +/** + * Initialization routine for run-time dependency on rdma-core. + */ +static int +mlx4_glue_init(void) +{ + const char *path[] = { + /* + * A basic security check is necessary before trusting + * MLX4_GLUE_PATH, which may override RTE_EAL_PMD_PATH. + */ + (geteuid() == getuid() && getegid() == getgid() ? + getenv("MLX4_GLUE_PATH") : NULL), + RTE_EAL_PMD_PATH, + }; + unsigned int i = 0; + void *handle = NULL; + void **sym; + const char *dlmsg; + + while (!handle && i != RTE_DIM(path)) { + const char *end; + size_t len; + int ret; + + if (!path[i]) { + ++i; + continue; + } + end = strpbrk(path[i], ":;"); + if (!end) + end = path[i] + strlen(path[i]); + len = end - path[i]; + ret = 0; + do { + char name[ret + 1]; + + ret = snprintf(name, sizeof(name), "%.*s%s" MLX4_GLUE, + (int)len, path[i], + (!len || *(end - 1) == '/') ? "" : "/"); + if (ret == -1) + break; + if (sizeof(name) != (size_t)ret + 1) + continue; + DEBUG("looking for rdma-core glue as \"%s\"", name); + handle = dlopen(name, RTLD_LAZY); + break; + } while (1); + path[i] = end + 1; + if (!*end) + ++i; + } + if (!handle) { + rte_errno = EINVAL; + dlmsg = dlerror(); + if (dlmsg) + WARN("cannot load glue library: %s", dlmsg); + goto glue_error; + } + sym = dlsym(handle, "mlx4_glue"); + if (!sym || !*sym) { + rte_errno = EINVAL; + dlmsg = dlerror(); + if (dlmsg) + ERROR("cannot resolve glue symbol: %s", dlmsg); + goto glue_error; + } + mlx4_glue = *sym; + return 0; +glue_error: + if (handle) + dlclose(handle); + WARN("cannot initialize PMD due to missing run-time" + " dependency on rdma-core libraries (libibverbs," + " libmlx4)"); + return -rte_errno; +} + +#endif + /** * Driver initialization routine. */ @@ -5916,7 +6010,26 @@ rte_mlx4_pmd_init(void) * using this PMD, which is not supported in forked processes. */ setenv("RDMAV_HUGEPAGES_SAFE", "1", 1); - ibv_fork_init(); +#ifdef RTE_LIBRTE_MLX4_DLOPEN_DEPS + if (mlx4_glue_init()) + return; + assert(mlx4_glue); +#endif +#ifndef NDEBUG + /* Glue structure must not contain any NULL pointers. */ + { + unsigned int i; + + for (i = 0; i != sizeof(*mlx4_glue) / sizeof(void *); ++i) + assert(((const void *const *)mlx4_glue)[i]); + } +#endif + if (strcmp(mlx4_glue->version, MLX4_GLUE_VERSION)) { + ERROR("rdma-core glue \"%s\" mismatch: \"%s\" is required", + mlx4_glue->version, MLX4_GLUE_VERSION); + return; + } + mlx4_glue->fork_init(); rte_eal_pci_register(&mlx4_driver.pci_drv); } diff --git a/drivers/net/mlx4/mlx4_glue.c b/drivers/net/mlx4/mlx4_glue.c new file mode 100644 index 0000000000..9634ddf514 --- /dev/null +++ b/drivers/net/mlx4/mlx4_glue.c @@ -0,0 +1,256 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2018 6WIND S.A. + * Copyright 2018 Mellanox + */ + +#include +#include + +/* Verbs headers do not support -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#include "mlx4_glue.h" + +static void +mlx4_glue_ack_async_event(struct ibv_async_event *event) +{ + ibv_ack_async_event(event); +} + +static struct ibv_pd * +mlx4_glue_alloc_pd(struct ibv_context *context) +{ + return ibv_alloc_pd(context); +} + +static int +mlx4_glue_close_device(struct ibv_context *context) +{ + return ibv_close_device(context); +} + +static struct ibv_flow * +mlx4_glue_create_flow(struct ibv_qp *qp, struct ibv_flow_attr *flow) +{ + return ibv_create_flow(qp, flow); +} + +static int +mlx4_glue_dealloc_pd(struct ibv_pd *pd) +{ + return ibv_dealloc_pd(pd); +} + +static int +mlx4_glue_dereg_mr(struct ibv_mr *mr) +{ + return ibv_dereg_mr(mr); +} + +static int +mlx4_glue_destroy_cq(struct ibv_cq *cq) +{ + return ibv_destroy_cq(cq); +} + +static int +mlx4_glue_destroy_flow(struct ibv_flow *flow_id) +{ + return ibv_destroy_flow(flow_id); +} + +static int +mlx4_glue_destroy_qp(struct ibv_qp *qp) +{ + return ibv_destroy_qp(qp); +} + +static struct ibv_cq * +mlx4_glue_exp_create_cq(struct ibv_context *context, int cqe, void *cq_context, + struct ibv_comp_channel *channel, int comp_vector, + struct ibv_exp_cq_init_attr *attr) +{ + return ibv_exp_create_cq(context, cqe, cq_context, channel, + comp_vector, attr); +} + +static struct ibv_qp * +mlx4_glue_exp_create_qp(struct ibv_context *context, + struct ibv_exp_qp_init_attr *qp_init_attr) +{ + return ibv_exp_create_qp(context, qp_init_attr); +} + +static struct ibv_exp_res_domain * +mlx4_glue_exp_create_res_domain(struct ibv_context *context, + struct ibv_exp_res_domain_init_attr *attr) +{ + return ibv_exp_create_res_domain(context, attr); +} + +static int +mlx4_glue_exp_destroy_res_domain(struct ibv_context *context, + struct ibv_exp_res_domain *res_dom, + struct ibv_exp_destroy_res_domain_attr *attr) +{ + return ibv_exp_destroy_res_domain(context, res_dom, attr); +} + +static int +mlx4_glue_exp_modify_qp(struct ibv_qp *qp, struct ibv_exp_qp_attr *attr, + uint64_t exp_attr_mask) +{ + return ibv_exp_modify_qp(qp, attr, exp_attr_mask); +} + +static int +mlx4_glue_exp_query_device(struct ibv_context *context, + struct ibv_exp_device_attr *attr) +{ + return ibv_exp_query_device(context, attr); +} + +static void * +mlx4_glue_exp_query_intf(struct ibv_context *context, + struct ibv_exp_query_intf_params *params, + enum ibv_exp_query_intf_status *status) +{ + return ibv_exp_query_intf(context, params, status); +} + +static int +mlx4_glue_exp_release_intf(struct ibv_context *context, void *intf, + struct ibv_exp_release_intf_params *params) +{ + return ibv_exp_release_intf(context, intf, params); +} + +static int +mlx4_glue_fork_init(void) +{ + return ibv_fork_init(); +} + +static void +mlx4_glue_free_device_list(struct ibv_device **list) +{ + ibv_free_device_list(list); +} + +static int +mlx4_glue_get_async_event(struct ibv_context *context, + struct ibv_async_event *event) +{ + return ibv_get_async_event(context, event); +} + +static struct ibv_device ** +mlx4_glue_get_device_list(int *num_devices) +{ + return ibv_get_device_list(num_devices); +} + +static const char * +mlx4_glue_get_device_name(struct ibv_device *device) +{ + return ibv_get_device_name(device); +} + +static struct ibv_context * +mlx4_glue_open_device(struct ibv_device *device) +{ + return ibv_open_device(device); +} + +static int +mlx4_glue_poll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc) +{ + return ibv_poll_cq(cq, num_entries, wc); +} + +static const char * +mlx4_glue_port_state_str(enum ibv_port_state port_state) +{ + return ibv_port_state_str(port_state); +} + +static int +mlx4_glue_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr, + struct ibv_recv_wr **bad_wr) +{ + return ibv_post_recv(qp, wr, bad_wr); +} + +static int +mlx4_glue_query_device(struct ibv_context *context, + struct ibv_device_attr *device_attr) +{ + return ibv_query_device(context, device_attr); +} + +static int +mlx4_glue_query_port(struct ibv_context *context, uint8_t port_num, + struct ibv_port_attr *port_attr) +{ + return ibv_query_port(context, port_num, port_attr); +} + +static struct ibv_mr * +mlx4_glue_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access) +{ + return ibv_reg_mr(pd, addr, length, access); +} + +static int +mlx4_glue_resize_cq(struct ibv_cq *cq, int cqe) +{ + return ibv_resize_cq(cq, cqe); +} + +static const char * +mlx4_glue_wc_status_str(enum ibv_wc_status status) +{ + return ibv_wc_status_str(status); +} + +const struct mlx4_glue *mlx4_glue = &(const struct mlx4_glue){ + .version = MLX4_GLUE_VERSION, + .ack_async_event = mlx4_glue_ack_async_event, + .alloc_pd = mlx4_glue_alloc_pd, + .close_device = mlx4_glue_close_device, + .create_flow = mlx4_glue_create_flow, + .dealloc_pd = mlx4_glue_dealloc_pd, + .dereg_mr = mlx4_glue_dereg_mr, + .destroy_cq = mlx4_glue_destroy_cq, + .destroy_flow = mlx4_glue_destroy_flow, + .destroy_qp = mlx4_glue_destroy_qp, + .exp_create_cq = mlx4_glue_exp_create_cq, + .exp_create_qp = mlx4_glue_exp_create_qp, + .exp_create_res_domain = mlx4_glue_exp_create_res_domain, + .exp_destroy_res_domain = mlx4_glue_exp_destroy_res_domain, + .exp_modify_qp = mlx4_glue_exp_modify_qp, + .exp_query_device = mlx4_glue_exp_query_device, + .exp_query_intf = mlx4_glue_exp_query_intf, + .exp_release_intf = mlx4_glue_exp_release_intf, + .fork_init = mlx4_glue_fork_init, + .free_device_list = mlx4_glue_free_device_list, + .get_async_event = mlx4_glue_get_async_event, + .get_device_list = mlx4_glue_get_device_list, + .get_device_name = mlx4_glue_get_device_name, + .open_device = mlx4_glue_open_device, + .poll_cq = mlx4_glue_poll_cq, + .port_state_str = mlx4_glue_port_state_str, + .post_recv = mlx4_glue_post_recv, + .query_device = mlx4_glue_query_device, + .query_port = mlx4_glue_query_port, + .reg_mr = mlx4_glue_reg_mr, + .resize_cq = mlx4_glue_resize_cq, + .wc_status_str = mlx4_glue_wc_status_str, +}; diff --git a/drivers/net/mlx4/mlx4_glue.h b/drivers/net/mlx4/mlx4_glue.h new file mode 100644 index 0000000000..06e81e4949 --- /dev/null +++ b/drivers/net/mlx4/mlx4_glue.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2018 6WIND S.A. + * Copyright 2018 Mellanox + */ + +#ifndef MLX4_GLUE_H_ +#define MLX4_GLUE_H_ + +#include +#include + +/* Verbs headers do not support -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#ifndef MLX4_GLUE_VERSION +#define MLX4_GLUE_VERSION "" +#endif + +/* LIB_GLUE_VERSION must be updated every time this structure is modified. */ +struct mlx4_glue { + const char *version; + void (*ack_async_event)(struct ibv_async_event *event); + struct ibv_pd *(*alloc_pd)(struct ibv_context *context); + int (*close_device)(struct ibv_context *context); + struct ibv_flow *(*create_flow)(struct ibv_qp *qp, + struct ibv_flow_attr *flow); + int (*dealloc_pd)(struct ibv_pd *pd); + int (*dereg_mr)(struct ibv_mr *mr); + int (*destroy_cq)(struct ibv_cq *cq); + int (*destroy_flow)(struct ibv_flow *flow_id); + int (*destroy_qp)(struct ibv_qp *qp); + struct ibv_cq *(*exp_create_cq) + (struct ibv_context *context, int cqe, void *cq_context, + struct ibv_comp_channel *channel, int comp_vector, + struct ibv_exp_cq_init_attr *attr); + struct ibv_qp *(*exp_create_qp) + (struct ibv_context *context, + struct ibv_exp_qp_init_attr *qp_init_attr); + struct ibv_exp_res_domain *(*exp_create_res_domain) + (struct ibv_context *context, + struct ibv_exp_res_domain_init_attr *attr); + int (*exp_destroy_res_domain) + (struct ibv_context *context, + struct ibv_exp_res_domain *res_dom, + struct ibv_exp_destroy_res_domain_attr *attr); + int (*exp_modify_qp)(struct ibv_qp *qp, struct ibv_exp_qp_attr *attr, + uint64_t exp_attr_mask); + int (*exp_query_device)(struct ibv_context *context, + struct ibv_exp_device_attr *attr); + void *(*exp_query_intf)(struct ibv_context *context, + struct ibv_exp_query_intf_params *params, + enum ibv_exp_query_intf_status *status); + int (*exp_release_intf)(struct ibv_context *context, void *intf, + struct ibv_exp_release_intf_params *params); + int (*fork_init)(void); + void (*free_device_list)(struct ibv_device **list); + int (*get_async_event)(struct ibv_context *context, + struct ibv_async_event *event); + struct ibv_device **(*get_device_list)(int *num_devices); + const char *(*get_device_name)(struct ibv_device *device); + struct ibv_context *(*open_device)(struct ibv_device *device); + int (*poll_cq)(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc); + const char *(*port_state_str)(enum ibv_port_state port_state); + int (*post_recv)(struct ibv_qp *qp, struct ibv_recv_wr *wr, + struct ibv_recv_wr **bad_wr); + int (*query_device)(struct ibv_context *context, + struct ibv_device_attr *device_attr); + int (*query_port)(struct ibv_context *context, uint8_t port_num, + struct ibv_port_attr *port_attr); + struct ibv_mr *(*reg_mr)(struct ibv_pd *pd, void *addr, size_t length, + int access); + int (*resize_cq)(struct ibv_cq *cq, int cqe); + const char *(*wc_status_str)(enum ibv_wc_status status); +}; + +const struct mlx4_glue *mlx4_glue; + +#endif /* MLX4_GLUE_H_ */ diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile index 671089c3ff..35bfe9be73 100644 --- a/drivers/net/mlx5/Makefile +++ b/drivers/net/mlx5/Makefile @@ -33,9 +33,15 @@ include $(RTE_SDK)/mk/rte.vars.mk # Library name. LIB = librte_pmd_mlx5.a +LIB_GLUE = $(LIB_GLUE_BASE).$(LIB_GLUE_VERSION) +LIB_GLUE_BASE = librte_pmd_mlx5_glue.so +LIB_GLUE_VERSION = 17.02.1 # Sources. SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5.c +ifneq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y) +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_glue.c +endif SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxq.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_txq.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx.c @@ -57,6 +63,10 @@ DEPDIRS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/librte_eal DEPDIRS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/librte_mempool DEPDIRS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/librte_kvargs +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y) +INSTALL-$(CONFIG_RTE_LIBRTE_MLX5_PMD)-lib += $(LIB_GLUE) +endif + # Basic CFLAGS. CFLAGS += -O3 CFLAGS += -std=gnu99 -Wall -Wextra @@ -67,7 +77,14 @@ CFLAGS += -D_DEFAULT_SOURCE CFLAGS += -D_XOPEN_SOURCE=600 CFLAGS += $(WERROR_FLAGS) CFLAGS += -Wno-strict-prototypes +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y) +CFLAGS += -DMLX5_GLUE='"$(LIB_GLUE)"' +CFLAGS += -DMLX5_GLUE_VERSION='"$(LIB_GLUE_VERSION)"' +CFLAGS_mlx5_glue.o += -fPIC +LDLIBS += -ldl +else LDLIBS += -libverbs +endif # A few warnings cannot be avoided in external headers. CFLAGS += -Wno-error=cast-qual @@ -147,7 +164,24 @@ mlx5_autoconf.h: mlx5_autoconf.h.new $(SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD):.c=.o): mlx5_autoconf.h +# Generate dependency plug-in for rdma-core when the PMD must not be linked +# directly, so that applications do not inherit this dependency. + +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y) + +$(LIB): $(LIB_GLUE) + +$(LIB_GLUE): mlx5_glue.o + $Q $(LD) $(LDFLAGS) $(EXTRA_LDFLAGS) \ + -Wl,-h,$(LIB_GLUE) \ + -s -shared -o $@ $< -libverbs + +mlx5_glue.o: mlx5_autoconf.h + +endif + clean_mlx5: FORCE $Q rm -f -- mlx5_autoconf.h mlx5_autoconf.h.new + $Q rm -f -- mlx5_glue.o $(LIB_GLUE_BASE)* clean: clean_mlx5 diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index d4bd4696ce..c4a36b4f1e 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -58,12 +59,14 @@ #include #include #include +#include #include #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" +#include "mlx5_glue.h" #include "mlx5_utils.h" #include "mlx5_rxtx.h" #include "mlx5_autoconf.h" @@ -170,8 +173,8 @@ mlx5_dev_close(struct rte_eth_dev *dev) } if (priv->pd != NULL) { assert(priv->ctx != NULL); - claim_zero(ibv_dealloc_pd(priv->pd)); - claim_zero(ibv_close_device(priv->ctx)); + claim_zero(mlx5_glue->dealloc_pd(priv->pd)); + claim_zero(mlx5_glue->close_device(priv->ctx)); } else assert(priv->ctx == NULL); if (priv->rss_conf != NULL) { @@ -384,7 +387,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) /* Save PCI address. */ mlx5_dev[idx].pci_addr = pci_dev->addr; - list = ibv_get_device_list(&i); + list = mlx5_glue->get_device_list(&i); if (list == NULL) { assert(errno); if (errno == ENOSYS) { @@ -438,12 +441,12 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) list[i]->name, sriov ? "true" : "false", mps ? "true" : "false"); - attr_ctx = ibv_open_device(list[i]); + attr_ctx = mlx5_glue->open_device(list[i]); err = errno; break; } if (attr_ctx == NULL) { - ibv_free_device_list(list); + mlx5_glue->free_device_list(list); switch (err) { case 0: WARN("cannot access device, is mlx5_ib loaded?"); @@ -458,7 +461,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) ibv_dev = list[i]; DEBUG("device opened"); - if (ibv_query_device(attr_ctx, &device_attr)) + if (mlx5_glue->query_device(attr_ctx, &device_attr)) goto error; INFO("%u port(s) detected", device_attr.phys_port_cnt); @@ -483,12 +486,12 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) DEBUG("using port %u (%08" PRIx32 ")", port, test); - ctx = ibv_open_device(ibv_dev); + ctx = mlx5_glue->open_device(ibv_dev); if (ctx == NULL) goto port_error; /* Check port status. */ - err = ibv_query_port(ctx, port, &port_attr); + err = mlx5_glue->query_port(ctx, port, &port_attr); if (err) { ERROR("port query failed: %s", strerror(err)); goto port_error; @@ -502,11 +505,11 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) if (port_attr.state != IBV_PORT_ACTIVE) DEBUG("port %d is not active: \"%s\" (%d)", - port, ibv_port_state_str(port_attr.state), + port, mlx5_glue->port_state_str(port_attr.state), port_attr.state); /* Allocate protection domain. */ - pd = ibv_alloc_pd(ctx); + pd = mlx5_glue->alloc_pd(ctx); if (pd == NULL) { ERROR("PD allocation failure"); err = ENOMEM; @@ -538,7 +541,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) strerror(err)); goto port_error; } - if (ibv_exp_query_device(ctx, &exp_device_attr)) { + if (mlx5_glue->exp_query_device(ctx, &exp_device_attr)) { ERROR("ibv_exp_query_device() failed"); goto port_error; } @@ -638,7 +641,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) char name[RTE_ETH_NAME_MAX_LEN]; snprintf(name, sizeof(name), "%s port %u", - ibv_get_device_name(ibv_dev), port); + mlx5_glue->get_device_name(ibv_dev), port); eth_dev = rte_eth_dev_allocate(name); } if (eth_dev == NULL) { @@ -695,9 +698,9 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) rte_free(priv); } if (pd) - claim_zero(ibv_dealloc_pd(pd)); + claim_zero(mlx5_glue->dealloc_pd(pd)); if (ctx) - claim_zero(ibv_close_device(ctx)); + claim_zero(mlx5_glue->close_device(ctx)); break; } @@ -716,9 +719,9 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) error: if (attr_ctx) - claim_zero(ibv_close_device(attr_ctx)); + claim_zero(mlx5_glue->close_device(attr_ctx)); if (list) - ibv_free_device_list(list); + mlx5_glue->free_device_list(list); assert(err >= 0); return -err; } @@ -773,6 +776,88 @@ static struct eth_driver mlx5_driver = { .dev_private_size = sizeof(struct priv) }; +#ifdef RTE_LIBRTE_MLX5_DLOPEN_DEPS + +/** + * Initialization routine for run-time dependency on rdma-core. + */ +static int +mlx5_glue_init(void) +{ + const char *path[] = { + /* + * A basic security check is necessary before trusting + * MLX5_GLUE_PATH, which may override RTE_EAL_PMD_PATH. + */ + (geteuid() == getuid() && getegid() == getgid() ? + getenv("MLX5_GLUE_PATH") : NULL), + RTE_EAL_PMD_PATH, + }; + unsigned int i = 0; + void *handle = NULL; + void **sym; + const char *dlmsg; + + while (!handle && i != RTE_DIM(path)) { + const char *end; + size_t len; + int ret; + + if (!path[i]) { + ++i; + continue; + } + end = strpbrk(path[i], ":;"); + if (!end) + end = path[i] + strlen(path[i]); + len = end - path[i]; + ret = 0; + do { + char name[ret + 1]; + + ret = snprintf(name, sizeof(name), "%.*s%s" MLX5_GLUE, + (int)len, path[i], + (!len || *(end - 1) == '/') ? "" : "/"); + if (ret == -1) + break; + if (sizeof(name) != (size_t)ret + 1) + continue; + DEBUG("looking for rdma-core glue as \"%s\"", name); + handle = dlopen(name, RTLD_LAZY); + break; + } while (1); + path[i] = end + 1; + if (!*end) + ++i; + } + if (!handle) { + rte_errno = EINVAL; + dlmsg = dlerror(); + if (dlmsg) + WARN("cannot load glue library: %s", dlmsg); + goto glue_error; + } + sym = dlsym(handle, "mlx5_glue"); + if (!sym || !*sym) { + rte_errno = EINVAL; + dlmsg = dlerror(); + if (dlmsg) + ERROR("cannot resolve glue symbol: %s", dlmsg); + goto glue_error; + } + mlx5_glue = *sym; + return 0; +glue_error: + if (handle) + dlclose(handle); + WARN("cannot initialize PMD due to missing run-time" + " dependency on rdma-core libraries (libibverbs," + " libmlx5)"); + return -rte_errno; +} + +#endif + /** * Driver initialization routine. */ @@ -787,7 +872,26 @@ rte_mlx5_pmd_init(void) * using this PMD, which is not supported in forked processes. */ setenv("RDMAV_HUGEPAGES_SAFE", "1", 1); - ibv_fork_init(); +#ifdef RTE_LIBRTE_MLX5_DLOPEN_DEPS + if (mlx5_glue_init()) + return; + assert(mlx5_glue); +#endif +#ifndef NDEBUG + /* Glue structure must not contain any NULL pointers. */ + { + unsigned int i; + + for (i = 0; i != sizeof(*mlx5_glue) / sizeof(void *); ++i) + assert(((const void *const *)mlx5_glue)[i]); + } +#endif + if (strcmp(mlx5_glue->version, MLX5_GLUE_VERSION)) { + ERROR("rdma-core glue \"%s\" mismatch: \"%s\" is required", + mlx5_glue->version, MLX5_GLUE_VERSION); + return; + } + mlx5_glue->fork_init(); rte_eal_pci_register(&mlx5_driver.pci_drv); } diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c index 2145965f69..2fe14b5099 100644 --- a/drivers/net/mlx5/mlx5_ethdev.c +++ b/drivers/net/mlx5/mlx5_ethdev.c @@ -66,6 +66,7 @@ #endif #include "mlx5.h" +#include "mlx5_glue.h" #include "mlx5_rxtx.h" #include "mlx5_utils.h" @@ -1172,14 +1173,14 @@ priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev) /* Read all message and acknowledge them. */ for (;;) { - if (ibv_get_async_event(priv->ctx, &event)) + if (mlx5_glue->get_async_event(priv->ctx, &event)) break; if (event.event_type != IBV_EVENT_PORT_ACTIVE && event.event_type != IBV_EVENT_PORT_ERR) DEBUG("event type %d on port %d not handled", event.event_type, event.element.port_num); - ibv_ack_async_event(&event); + mlx5_glue->ack_async_event(&event); } mlx5_link_update(dev, 0); if (((link->link_speed == 0) && link->link_status) || diff --git a/drivers/net/mlx5/mlx5_fdir.c b/drivers/net/mlx5/mlx5_fdir.c index f80c58b4dd..a528fb2096 100644 --- a/drivers/net/mlx5/mlx5_fdir.c +++ b/drivers/net/mlx5/mlx5_fdir.c @@ -62,6 +62,7 @@ #endif #include "mlx5.h" +#include "mlx5_glue.h" #include "mlx5_rxtx.h" struct fdir_flow_desc { @@ -386,7 +387,7 @@ priv_fdir_flow_add(struct priv *priv, create_flow: errno = 0; - flow = ibv_exp_create_flow(fdir_queue->qp, attr); + flow = mlx5_glue->exp_create_flow(fdir_queue->qp, attr); if (flow == NULL) { /* It's not clear whether errno is always set in this case. */ ERROR("%p: flow director configuration failed, errno=%d: %s", @@ -422,18 +423,19 @@ priv_fdir_queue_destroy(struct priv *priv, struct fdir_queue *fdir_queue) assert(idx < priv->rxqs_n); if (fdir_queue == rxq_ctrl->fdir_queue && fdir_filter->flow != NULL) { - claim_zero(ibv_exp_destroy_flow(fdir_filter->flow)); + claim_zero(mlx5_glue->exp_destroy_flow + (fdir_filter->flow)); fdir_filter->flow = NULL; } } assert(fdir_queue->qp); - claim_zero(ibv_destroy_qp(fdir_queue->qp)); + claim_zero(mlx5_glue->destroy_qp(fdir_queue->qp)); assert(fdir_queue->ind_table); - claim_zero(ibv_exp_destroy_rwq_ind_table(fdir_queue->ind_table)); + claim_zero(mlx5_glue->exp_destroy_rwq_ind_table(fdir_queue->ind_table)); if (fdir_queue->wq) - claim_zero(ibv_exp_destroy_wq(fdir_queue->wq)); + claim_zero(mlx5_glue->exp_destroy_wq(fdir_queue->wq)); if (fdir_queue->cq) - claim_zero(ibv_destroy_cq(fdir_queue->cq)); + claim_zero(mlx5_glue->destroy_cq(fdir_queue->cq)); #ifndef NDEBUG memset(fdir_queue, 0x2a, sizeof(*fdir_queue)); #endif @@ -467,45 +469,45 @@ priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq, assert(priv->pd); assert(priv->ctx); if (!wq) { - fdir_queue->cq = ibv_exp_create_cq( - priv->ctx, 1, NULL, NULL, 0, - &(struct ibv_exp_cq_init_attr){ + fdir_queue->cq = mlx5_glue->exp_create_cq + (priv->ctx, 1, NULL, NULL, 0, + &(struct ibv_exp_cq_init_attr){ .comp_mask = 0, - }); + }); if (!fdir_queue->cq) { ERROR("cannot create flow director CQ"); goto error; } - fdir_queue->wq = ibv_exp_create_wq( - priv->ctx, - &(struct ibv_exp_wq_init_attr){ + fdir_queue->wq = mlx5_glue->exp_create_wq + (priv->ctx, + &(struct ibv_exp_wq_init_attr){ .wq_type = IBV_EXP_WQT_RQ, .max_recv_wr = 1, .max_recv_sge = 1, .pd = priv->pd, .cq = fdir_queue->cq, - }); + }); if (!fdir_queue->wq) { ERROR("cannot create flow director WQ"); goto error; } wq = fdir_queue->wq; } - fdir_queue->ind_table = ibv_exp_create_rwq_ind_table( - priv->ctx, - &(struct ibv_exp_rwq_ind_table_init_attr){ + fdir_queue->ind_table = mlx5_glue->exp_create_rwq_ind_table + (priv->ctx, + &(struct ibv_exp_rwq_ind_table_init_attr){ .pd = priv->pd, .log_ind_tbl_size = 0, .ind_tbl = &wq, .comp_mask = 0, - }); + }); if (!fdir_queue->ind_table) { ERROR("cannot create flow director indirection table"); goto error; } - fdir_queue->qp = ibv_exp_create_qp( - priv->ctx, - &(struct ibv_exp_qp_init_attr){ + fdir_queue->qp = mlx5_glue->exp_create_qp + (priv->ctx, + &(struct ibv_exp_qp_init_attr){ .qp_type = IBV_QPT_RAW_PACKET, .comp_mask = IBV_EXP_QP_INIT_ATTR_PD | @@ -521,7 +523,7 @@ priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq, .rwq_ind_tbl = fdir_queue->ind_table, }, .port_num = priv->port, - }); + }); if (!fdir_queue->qp) { ERROR("cannot create flow director hash RX QP"); goto error; @@ -531,12 +533,12 @@ priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq, assert(fdir_queue); assert(!fdir_queue->qp); if (fdir_queue->ind_table) - claim_zero(ibv_exp_destroy_rwq_ind_table + claim_zero(mlx5_glue->exp_destroy_rwq_ind_table (fdir_queue->ind_table)); if (fdir_queue->wq) - claim_zero(ibv_exp_destroy_wq(fdir_queue->wq)); + claim_zero(mlx5_glue->exp_destroy_wq(fdir_queue->wq)); if (fdir_queue->cq) - claim_zero(ibv_destroy_cq(fdir_queue->cq)); + claim_zero(mlx5_glue->destroy_cq(fdir_queue->cq)); rte_free(fdir_queue); return NULL; } @@ -685,7 +687,7 @@ priv_fdir_filter_flush(struct priv *priv) (void *)priv, (void *)mlx5_fdir_filter); LIST_REMOVE(mlx5_fdir_filter, next); if (flow != NULL) - claim_zero(ibv_exp_destroy_flow(flow)); + claim_zero(mlx5_glue->exp_destroy_flow(flow)); rte_free(mlx5_fdir_filter); } } @@ -726,7 +728,7 @@ priv_fdir_disable(struct priv *priv) /* Destroy flow handle */ if (flow != NULL) { - claim_zero(ibv_exp_destroy_flow(flow)); + claim_zero(mlx5_glue->exp_destroy_flow(flow)); mlx5_fdir_filter->flow = NULL; } } @@ -899,7 +901,7 @@ priv_fdir_filter_update(struct priv *priv, /* Destroy flow handle. */ if (flow != NULL) { - claim_zero(ibv_exp_destroy_flow(flow)); + claim_zero(mlx5_glue->exp_destroy_flow(flow)); mlx5_fdir_filter->flow = NULL; } DEBUG("%p: flow director filter %p updated", @@ -944,7 +946,7 @@ priv_fdir_filter_delete(struct priv *priv, /* Destroy flow handle. */ if (flow != NULL) { - claim_zero(ibv_exp_destroy_flow(flow)); + claim_zero(mlx5_glue->exp_destroy_flow(flow)); mlx5_fdir_filter->flow = NULL; } diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 23c1b5ef3c..1d6a2038fd 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -50,6 +50,7 @@ #include #include "mlx5.h" +#include "mlx5_glue.h" #include "mlx5_prm.h" static int @@ -883,25 +884,26 @@ priv_flow_create_action_queue(struct priv *priv, return NULL; } if (action->drop) { - rte_flow->cq = - ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0, - &(struct ibv_exp_cq_init_attr){ - .comp_mask = 0, - }); + rte_flow->cq = mlx5_glue->exp_create_cq + (priv->ctx, 1, NULL, NULL, 0, + &(struct ibv_exp_cq_init_attr){ + .comp_mask = 0, + }); if (!rte_flow->cq) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "cannot allocate CQ"); goto error; } - rte_flow->wq = ibv_exp_create_wq(priv->ctx, - &(struct ibv_exp_wq_init_attr){ - .wq_type = IBV_EXP_WQT_RQ, - .max_recv_wr = 1, - .max_recv_sge = 1, - .pd = priv->pd, - .cq = rte_flow->cq, - }); + rte_flow->wq = mlx5_glue->exp_create_wq + (priv->ctx, + &(struct ibv_exp_wq_init_attr){ + .wq_type = IBV_EXP_WQT_RQ, + .max_recv_wr = 1, + .max_recv_sge = 1, + .pd = priv->pd, + .cq = rte_flow->cq, + }); } else { rxq = container_of((*priv->rxqs)[action->queue_id], struct rxq_ctrl, rxq); @@ -911,22 +913,22 @@ priv_flow_create_action_queue(struct priv *priv, } rte_flow->mark = action->mark; rte_flow->ibv_attr = ibv_attr; - rte_flow->ind_table = ibv_exp_create_rwq_ind_table( - priv->ctx, - &(struct ibv_exp_rwq_ind_table_init_attr){ + rte_flow->ind_table = mlx5_glue->exp_create_rwq_ind_table + (priv->ctx, + &(struct ibv_exp_rwq_ind_table_init_attr){ .pd = priv->pd, .log_ind_tbl_size = 0, .ind_tbl = &rte_flow->wq, .comp_mask = 0, - }); + }); if (!rte_flow->ind_table) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "cannot allocate indirection table"); goto error; } - rte_flow->qp = ibv_exp_create_qp( - priv->ctx, - &(struct ibv_exp_qp_init_attr){ + rte_flow->qp = mlx5_glue->exp_create_qp + (priv->ctx, + &(struct ibv_exp_qp_init_attr){ .qp_type = IBV_QPT_RAW_PACKET, .comp_mask = IBV_EXP_QP_INIT_ATTR_PD | @@ -942,7 +944,7 @@ priv_flow_create_action_queue(struct priv *priv, .rwq_ind_tbl = rte_flow->ind_table, }, .port_num = priv->port, - }); + }); if (!rte_flow->qp) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "cannot allocate QP"); @@ -950,8 +952,8 @@ priv_flow_create_action_queue(struct priv *priv, } if (!priv->started) return rte_flow; - rte_flow->ibv_flow = ibv_exp_create_flow(rte_flow->qp, - rte_flow->ibv_attr); + rte_flow->ibv_flow = mlx5_glue->exp_create_flow(rte_flow->qp, + rte_flow->ibv_attr); if (!rte_flow->ibv_flow) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "flow rule creation failure"); @@ -961,14 +963,13 @@ priv_flow_create_action_queue(struct priv *priv, error: assert(rte_flow); if (rte_flow->qp) - ibv_destroy_qp(rte_flow->qp); + mlx5_glue->destroy_qp(rte_flow->qp); if (rte_flow->ind_table) - ibv_exp_destroy_rwq_ind_table(rte_flow->ind_table); + mlx5_glue->exp_destroy_rwq_ind_table(rte_flow->ind_table); if (!rte_flow->rxq && rte_flow->wq) - ibv_exp_destroy_wq(rte_flow->wq); + mlx5_glue->exp_destroy_wq(rte_flow->wq); if (!rte_flow->rxq && rte_flow->cq) - ibv_destroy_cq(rte_flow->cq); - rte_free(rte_flow->ibv_attr); + mlx5_glue->destroy_cq(rte_flow->cq); rte_free(rte_flow); return NULL; } @@ -1109,15 +1110,16 @@ priv_flow_destroy(struct priv *priv, (void)priv; LIST_REMOVE(flow, next); if (flow->ibv_flow) - claim_zero(ibv_exp_destroy_flow(flow->ibv_flow)); + claim_zero(mlx5_glue->exp_destroy_flow(flow->ibv_flow)); if (flow->qp) - claim_zero(ibv_destroy_qp(flow->qp)); + claim_zero(mlx5_glue->destroy_qp(flow->qp)); if (flow->ind_table) - claim_zero(ibv_exp_destroy_rwq_ind_table(flow->ind_table)); + claim_zero(mlx5_glue->exp_destroy_rwq_ind_table + (flow->ind_table)); if (!flow->rxq && flow->wq) - claim_zero(ibv_exp_destroy_wq(flow->wq)); + claim_zero(mlx5_glue->exp_destroy_wq(flow->wq)); if (!flow->rxq && flow->cq) - claim_zero(ibv_destroy_cq(flow->cq)); + claim_zero(mlx5_glue->destroy_cq(flow->cq)); if (flow->mark) { struct rte_flow *tmp; uint32_t mark_n = 0; @@ -1207,7 +1209,7 @@ priv_flow_stop(struct priv *priv) for (flow = LIST_FIRST(&priv->flows); flow; flow = LIST_NEXT(flow, next)) { - claim_zero(ibv_exp_destroy_flow(flow->ibv_flow)); + claim_zero(mlx5_glue->exp_destroy_flow(flow->ibv_flow)); flow->ibv_flow = NULL; if (flow->mark) flow->rxq->mark = 0; @@ -1232,8 +1234,8 @@ priv_flow_start(struct priv *priv) for (flow = LIST_FIRST(&priv->flows); flow; flow = LIST_NEXT(flow, next)) { - flow->ibv_flow = ibv_exp_create_flow(flow->qp, - flow->ibv_attr); + flow->ibv_flow = mlx5_glue->exp_create_flow(flow->qp, + flow->ibv_attr); if (!flow->ibv_flow) { DEBUG("Flow %p cannot be applied", (void *)flow); rte_errno = EINVAL; diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c new file mode 100644 index 0000000000..347503c30c --- /dev/null +++ b/drivers/net/mlx5/mlx5_glue.c @@ -0,0 +1,263 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2018 6WIND S.A. + * Copyright 2018 Mellanox Technologies, Ltd. + */ + +#include +#include + +/* Verbs headers do not support -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#include "mlx5_glue.h" + +static void +mlx5_glue_ack_async_event(struct ibv_async_event *event) +{ + ibv_ack_async_event(event); +} + +static struct ibv_pd * +mlx5_glue_alloc_pd(struct ibv_context *context) +{ + return ibv_alloc_pd(context); +} + +static int +mlx5_glue_close_device(struct ibv_context *context) +{ + return ibv_close_device(context); +} + +static int +mlx5_glue_dealloc_pd(struct ibv_pd *pd) +{ + return ibv_dealloc_pd(pd); +} + +static int +mlx5_glue_dereg_mr(struct ibv_mr *mr) +{ + return ibv_dereg_mr(mr); +} + +static int +mlx5_glue_destroy_cq(struct ibv_cq *cq) +{ + return ibv_destroy_cq(cq); +} + +static int +mlx5_glue_destroy_qp(struct ibv_qp *qp) +{ + return ibv_destroy_qp(qp); +} + +static struct ibv_cq * +mlx5_glue_exp_create_cq(struct ibv_context *context, int cqe, void *cq_context, + struct ibv_comp_channel *channel, int comp_vector, + struct ibv_exp_cq_init_attr *attr) +{ + return ibv_exp_create_cq(context, cqe, cq_context, channel, + comp_vector, attr); +} + +static struct ibv_exp_flow * +mlx5_glue_exp_create_flow(struct ibv_qp *qp, struct ibv_exp_flow_attr *flow) +{ + return ibv_exp_create_flow(qp, flow); +} + +static struct ibv_qp * +mlx5_glue_exp_create_qp(struct ibv_context *context, + struct ibv_exp_qp_init_attr *qp_init_attr) +{ + return ibv_exp_create_qp(context, qp_init_attr); +} + +static struct ibv_exp_res_domain * +mlx5_glue_exp_create_res_domain(struct ibv_context *context, + struct ibv_exp_res_domain_init_attr *attr) +{ + return ibv_exp_create_res_domain(context, attr); +} + +static struct ibv_exp_rwq_ind_table * +mlx5_glue_exp_create_rwq_ind_table(struct ibv_context *context, + struct ibv_exp_rwq_ind_table_init_attr *init_attr) +{ + return ibv_exp_create_rwq_ind_table(context, init_attr); +} + +static struct ibv_exp_wq * +mlx5_glue_exp_create_wq(struct ibv_context *context, + struct ibv_exp_wq_init_attr *wq_init_attr) +{ + return ibv_exp_create_wq(context, wq_init_attr); +} + +static int +mlx5_glue_exp_destroy_flow(struct ibv_exp_flow *flow_id) +{ + return ibv_exp_destroy_flow(flow_id); +} + +static int +mlx5_glue_exp_destroy_res_domain(struct ibv_context *context, + struct ibv_exp_res_domain *res_dom, + struct ibv_exp_destroy_res_domain_attr *attr) +{ + return ibv_exp_destroy_res_domain(context, res_dom, attr); +} + +static int +mlx5_glue_exp_destroy_rwq_ind_table(struct ibv_exp_rwq_ind_table *rwq_ind_table) +{ + return ibv_exp_destroy_rwq_ind_table(rwq_ind_table); +} + +static int +mlx5_glue_exp_destroy_wq(struct ibv_exp_wq *wq) +{ + return ibv_exp_destroy_wq(wq); +} + +static int +mlx5_glue_exp_modify_qp(struct ibv_qp *qp, struct ibv_exp_qp_attr *attr, + uint64_t exp_attr_mask) +{ + return ibv_exp_modify_qp(qp, attr, exp_attr_mask); +} + +static int +mlx5_glue_exp_modify_wq(struct ibv_exp_wq *wq, struct ibv_exp_wq_attr *wq_attr) +{ + return ibv_exp_modify_wq(wq, wq_attr); +} + +static int +mlx5_glue_exp_query_device(struct ibv_context *context, + struct ibv_exp_device_attr *attr) +{ + return ibv_exp_query_device(context, attr); +} + +static void * +mlx5_glue_exp_query_intf(struct ibv_context *context, + struct ibv_exp_query_intf_params *params, + enum ibv_exp_query_intf_status *status) +{ + return ibv_exp_query_intf(context, params, status); +} + +static int +mlx5_glue_exp_release_intf(struct ibv_context *context, void *intf, + struct ibv_exp_release_intf_params *params) +{ + return ibv_exp_release_intf(context, intf, params); +} + +static int +mlx5_glue_fork_init(void) +{ + return ibv_fork_init(); +} + +static void +mlx5_glue_free_device_list(struct ibv_device **list) +{ + ibv_free_device_list(list); +} + +static int +mlx5_glue_get_async_event(struct ibv_context *context, + struct ibv_async_event *event) +{ + return ibv_get_async_event(context, event); +} + +static struct ibv_device ** +mlx5_glue_get_device_list(int *num_devices) +{ + return ibv_get_device_list(num_devices); +} + +static const char * +mlx5_glue_get_device_name(struct ibv_device *device) +{ + return ibv_get_device_name(device); +} + +static struct ibv_context * +mlx5_glue_open_device(struct ibv_device *device) +{ + return ibv_open_device(device); +} + +static const char * +mlx5_glue_port_state_str(enum ibv_port_state port_state) +{ + return ibv_port_state_str(port_state); +} + +static int +mlx5_glue_query_device(struct ibv_context *context, + struct ibv_device_attr *device_attr) +{ + return ibv_query_device(context, device_attr); +} + +static int +mlx5_glue_query_port(struct ibv_context *context, uint8_t port_num, + struct ibv_port_attr *port_attr) +{ + return ibv_query_port(context, port_num, port_attr); +} + +static struct ibv_mr * +mlx5_glue_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access) +{ + return ibv_reg_mr(pd, addr, length, access); +} + +const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){ + .version = MLX5_GLUE_VERSION, + .ack_async_event = mlx5_glue_ack_async_event, + .alloc_pd = mlx5_glue_alloc_pd, + .close_device = mlx5_glue_close_device, + .dealloc_pd = mlx5_glue_dealloc_pd, + .dereg_mr = mlx5_glue_dereg_mr, + .destroy_cq = mlx5_glue_destroy_cq, + .destroy_qp = mlx5_glue_destroy_qp, + .exp_create_cq = mlx5_glue_exp_create_cq, + .exp_create_flow = mlx5_glue_exp_create_flow, + .exp_create_qp = mlx5_glue_exp_create_qp, + .exp_create_res_domain = mlx5_glue_exp_create_res_domain, + .exp_create_rwq_ind_table = mlx5_glue_exp_create_rwq_ind_table, + .exp_create_wq = mlx5_glue_exp_create_wq, + .exp_destroy_flow = mlx5_glue_exp_destroy_flow, + .exp_destroy_res_domain = mlx5_glue_exp_destroy_res_domain, + .exp_destroy_rwq_ind_table = mlx5_glue_exp_destroy_rwq_ind_table, + .exp_destroy_wq = mlx5_glue_exp_destroy_wq, + .exp_modify_qp = mlx5_glue_exp_modify_qp, + .exp_modify_wq = mlx5_glue_exp_modify_wq, + .exp_query_device = mlx5_glue_exp_query_device, + .exp_query_intf = mlx5_glue_exp_query_intf, + .exp_release_intf = mlx5_glue_exp_release_intf, + .fork_init = mlx5_glue_fork_init, + .free_device_list = mlx5_glue_free_device_list, + .get_async_event = mlx5_glue_get_async_event, + .get_device_list = mlx5_glue_get_device_list, + .get_device_name = mlx5_glue_get_device_name, + .open_device = mlx5_glue_open_device, + .port_state_str = mlx5_glue_port_state_str, + .query_device = mlx5_glue_query_device, + .query_port = mlx5_glue_query_port, + .reg_mr = mlx5_glue_reg_mr, +}; diff --git a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h new file mode 100644 index 0000000000..d1842d9d35 --- /dev/null +++ b/drivers/net/mlx5/mlx5_glue.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2018 6WIND S.A. + * Copyright 2018 Mellanox Technologies, Ltd. + */ + +#ifndef MLX5_GLUE_H_ +#define MLX5_GLUE_H_ + +#include +#include + +/* Verbs headers do not support -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif +#ifndef MLX5_GLUE_VERSION +#define MLX5_GLUE_VERSION "" +#endif + +/* LIB_GLUE_VERSION must be updated every time this structure is modified. */ +struct mlx5_glue { + const char *version; + void (*ack_async_event)(struct ibv_async_event *event); + struct ibv_pd *(*alloc_pd)(struct ibv_context *context); + int (*close_device)(struct ibv_context *context); + int (*dealloc_pd)(struct ibv_pd *pd); + int (*dereg_mr)(struct ibv_mr *mr); + int (*destroy_cq)(struct ibv_cq *cq); + int (*destroy_qp)(struct ibv_qp *qp); + struct ibv_cq *(*exp_create_cq) + (struct ibv_context *context, int cqe, void *cq_context, + struct ibv_comp_channel *channel, int comp_vector, + struct ibv_exp_cq_init_attr *attr); + struct ibv_exp_flow *(*exp_create_flow) + (struct ibv_qp *qp, struct ibv_exp_flow_attr *flow); + struct ibv_qp *(*exp_create_qp) + (struct ibv_context *context, + struct ibv_exp_qp_init_attr *qp_init_attr); + struct ibv_exp_res_domain *(*exp_create_res_domain) + (struct ibv_context *context, + struct ibv_exp_res_domain_init_attr *attr); + struct ibv_exp_rwq_ind_table *(*exp_create_rwq_ind_table) + (struct ibv_context *context, + struct ibv_exp_rwq_ind_table_init_attr *init_attr); + struct ibv_exp_wq *(*exp_create_wq) + (struct ibv_context *context, + struct ibv_exp_wq_init_attr *wq_init_attr); + int (*exp_destroy_flow)(struct ibv_exp_flow *flow_id); + int (*exp_destroy_res_domain) + (struct ibv_context *context, + struct ibv_exp_res_domain *res_dom, + struct ibv_exp_destroy_res_domain_attr *attr); + int (*exp_destroy_rwq_ind_table) + (struct ibv_exp_rwq_ind_table *rwq_ind_table); + int (*exp_destroy_wq)(struct ibv_exp_wq *wq); + int (*exp_modify_qp)(struct ibv_qp *qp, struct ibv_exp_qp_attr *attr, + uint64_t exp_attr_mask); + int (*exp_modify_wq)(struct ibv_exp_wq *wq, + struct ibv_exp_wq_attr *wq_attr); + int (*exp_query_device)(struct ibv_context *context, + struct ibv_exp_device_attr *attr); + void *(*exp_query_intf)(struct ibv_context *context, + struct ibv_exp_query_intf_params *params, + enum ibv_exp_query_intf_status *status); + int (*exp_release_intf)(struct ibv_context *context, void *intf, + struct ibv_exp_release_intf_params *params); + int (*fork_init)(void); + void (*free_device_list)(struct ibv_device **list); + int (*get_async_event)(struct ibv_context *context, + struct ibv_async_event *event); + struct ibv_device **(*get_device_list)(int *num_devices); + const char *(*get_device_name)(struct ibv_device *device); + struct ibv_context *(*open_device)(struct ibv_device *device); + const char *(*port_state_str)(enum ibv_port_state port_state); + int (*query_device)(struct ibv_context *context, + struct ibv_device_attr *device_attr); + int (*query_port)(struct ibv_context *context, uint8_t port_num, + struct ibv_port_attr *port_attr); + struct ibv_mr *(*reg_mr)(struct ibv_pd *pd, void *addr, size_t length, + int access); +}; + +const struct mlx5_glue *mlx5_glue; + +#endif /* MLX5_GLUE_H_ */ diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c index 4fcfd3b880..bfa9ef8783 100644 --- a/drivers/net/mlx5/mlx5_mac.c +++ b/drivers/net/mlx5/mlx5_mac.c @@ -63,6 +63,7 @@ #endif #include "mlx5.h" +#include "mlx5_glue.h" #include "mlx5_utils.h" #include "mlx5_rxtx.h" #include "mlx5_defs.h" @@ -119,8 +120,8 @@ hash_rxq_del_mac_flow(struct hash_rxq *hash_rxq, unsigned int mac_index, (*mac)[0], (*mac)[1], (*mac)[2], (*mac)[3], (*mac)[4], (*mac)[5], mac_index, vlan_index); - claim_zero(ibv_exp_destroy_flow(hash_rxq->mac_flow - [mac_index][vlan_index])); + claim_zero(mlx5_glue->exp_destroy_flow(hash_rxq->mac_flow + [mac_index][vlan_index])); hash_rxq->mac_flow[mac_index][vlan_index] = NULL; } @@ -287,7 +288,7 @@ hash_rxq_add_mac_flow(struct hash_rxq *hash_rxq, unsigned int mac_index, vlan_id); /* Create related flow. */ errno = 0; - flow = ibv_exp_create_flow(hash_rxq->qp, attr); + flow = mlx5_glue->exp_create_flow(hash_rxq->qp, attr); if (flow == NULL) { /* It's not clear whether errno is always set in this case. */ ERROR("%p: flow configuration failed, errno=%d: %s", diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c index 0a3638460b..e7e658568c 100644 --- a/drivers/net/mlx5/mlx5_mr.c +++ b/drivers/net/mlx5/mlx5_mr.c @@ -51,6 +51,7 @@ #endif #include "mlx5.h" +#include "mlx5_glue.h" #include "mlx5_rxtx.h" struct mlx5_check_mempool_data { @@ -159,10 +160,10 @@ mlx5_mp2mr(struct ibv_pd *pd, struct rte_mempool *mp) DEBUG("mempool %p using start=%p end=%p size=%zu for MR", (void *)mp, (void *)start, (void *)end, (size_t)(end - start)); - return ibv_reg_mr(pd, - (void *)start, - end - start, - IBV_ACCESS_LOCAL_WRITE); + return mlx5_glue->reg_mr(pd, + (void *)start, + end - start, + IBV_ACCESS_LOCAL_WRITE); } /** @@ -201,7 +202,7 @@ txq_mp2mr_reg(struct txq *txq, struct rte_mempool *mp, unsigned int idx) DEBUG("%p: MR <-> MP table full, dropping oldest entry.", (void *)txq_ctrl); --idx; - claim_zero(ibv_dereg_mr(txq_ctrl->txq.mp2mr[0].mr)); + claim_zero(mlx5_glue->dereg_mr(txq_ctrl->txq.mp2mr[0].mr)); memmove(&txq_ctrl->txq.mp2mr[0], &txq_ctrl->txq.mp2mr[1], (sizeof(txq_ctrl->txq.mp2mr) - sizeof(txq_ctrl->txq.mp2mr[0]))); diff --git a/drivers/net/mlx5/mlx5_rxmode.c b/drivers/net/mlx5/mlx5_rxmode.c index 173e6e842b..b643eedeed 100644 --- a/drivers/net/mlx5/mlx5_rxmode.c +++ b/drivers/net/mlx5/mlx5_rxmode.c @@ -55,6 +55,7 @@ #endif #include "mlx5.h" +#include "mlx5_glue.h" #include "mlx5_rxtx.h" #include "mlx5_utils.h" @@ -178,7 +179,7 @@ hash_rxq_special_flow_enable_vlan(struct hash_rxq *hash_rxq, }; errno = 0; - flow = ibv_exp_create_flow(hash_rxq->qp, attr); + flow = mlx5_glue->exp_create_flow(hash_rxq->qp, attr); if (flow == NULL) { /* It's not clear whether errno is always set in this case. */ ERROR("%p: flow configuration failed, errno=%d: %s", @@ -215,7 +216,7 @@ hash_rxq_special_flow_disable_vlan(struct hash_rxq *hash_rxq, if (flow == NULL) return; - claim_zero(ibv_exp_destroy_flow(flow)); + claim_zero(mlx5_glue->exp_destroy_flow(flow)); hash_rxq->special_flow[flow_type][vlan_index] = NULL; DEBUG("%p: special flow %s (index %d) VLAN %u (index %u) disabled", (void *)hash_rxq, hash_rxq_flow_type_str(flow_type), flow_type, diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index 28e93d3ee0..0b47ddc420 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -62,6 +62,7 @@ #endif #include "mlx5.h" +#include "mlx5_glue.h" #include "mlx5_rxtx.h" #include "mlx5_utils.h" #include "mlx5_autoconf.h" @@ -410,8 +411,8 @@ priv_create_hash_rxqs(struct priv *priv) ind_tbl_size = priv->reta_idx_n; ind_init_attr.log_ind_tbl_size = log2above(ind_tbl_size); errno = 0; - ind_table = ibv_exp_create_rwq_ind_table(priv->ctx, - &ind_init_attr); + ind_table = mlx5_glue->exp_create_rwq_ind_table(priv->ctx, + &ind_init_attr); if (ind_table != NULL) { (*ind_tables)[i] = ind_table; continue; @@ -464,7 +465,8 @@ priv_create_hash_rxqs(struct priv *priv) j, i, type); *hash_rxq = (struct hash_rxq){ .priv = priv, - .qp = ibv_exp_create_qp(priv->ctx, &qp_init_attr), + .qp = mlx5_glue->exp_create_qp(priv->ctx, + &qp_init_attr), .type = type, }; if (hash_rxq->qp == NULL) { @@ -493,7 +495,7 @@ priv_create_hash_rxqs(struct priv *priv) if (qp == NULL) continue; - claim_zero(ibv_destroy_qp(qp)); + claim_zero(mlx5_glue->destroy_qp(qp)); } rte_free(hash_rxqs); } @@ -504,7 +506,8 @@ priv_create_hash_rxqs(struct priv *priv) if (ind_table == NULL) continue; - claim_zero(ibv_exp_destroy_rwq_ind_table(ind_table)); + claim_zero(mlx5_glue->exp_destroy_rwq_ind_table + (ind_table)); } rte_free(ind_tables); } @@ -543,7 +546,7 @@ priv_destroy_hash_rxqs(struct priv *priv) for (j = 0; (j != RTE_DIM(hash_rxq->mac_flow)); ++j) for (k = 0; (k != RTE_DIM(hash_rxq->mac_flow[j])); ++k) assert(hash_rxq->mac_flow[j][k] == NULL); - claim_zero(ibv_destroy_qp(hash_rxq->qp)); + claim_zero(mlx5_glue->destroy_qp(hash_rxq->qp)); } priv->hash_rxqs_n = 0; rte_free(priv->hash_rxqs); @@ -553,7 +556,7 @@ priv_destroy_hash_rxqs(struct priv *priv) (*priv->ind_tables)[i]; assert(ind_table != NULL); - claim_zero(ibv_exp_destroy_rwq_ind_table(ind_table)); + claim_zero(mlx5_glue->exp_destroy_rwq_ind_table(ind_table)); } priv->ind_tables_n = 0; rte_free(priv->ind_tables); @@ -754,9 +757,9 @@ rxq_cleanup(struct rxq_ctrl *rxq_ctrl) params = (struct ibv_exp_release_intf_params){ .comp_mask = 0, }; - claim_zero(ibv_exp_release_intf(rxq_ctrl->priv->ctx, - rxq_ctrl->if_wq, - ¶ms)); + claim_zero(mlx5_glue->exp_release_intf(rxq_ctrl->priv->ctx, + rxq_ctrl->if_wq, + ¶ms)); } if (rxq_ctrl->if_cq != NULL) { assert(rxq_ctrl->priv != NULL); @@ -765,14 +768,14 @@ rxq_cleanup(struct rxq_ctrl *rxq_ctrl) params = (struct ibv_exp_release_intf_params){ .comp_mask = 0, }; - claim_zero(ibv_exp_release_intf(rxq_ctrl->priv->ctx, - rxq_ctrl->if_cq, - ¶ms)); + claim_zero(mlx5_glue->exp_release_intf(rxq_ctrl->priv->ctx, + rxq_ctrl->if_cq, + ¶ms)); } if (rxq_ctrl->wq != NULL) - claim_zero(ibv_exp_destroy_wq(rxq_ctrl->wq)); + claim_zero(mlx5_glue->exp_destroy_wq(rxq_ctrl->wq)); if (rxq_ctrl->cq != NULL) - claim_zero(ibv_destroy_cq(rxq_ctrl->cq)); + claim_zero(mlx5_glue->destroy_cq(rxq_ctrl->cq)); if (rxq_ctrl->rd != NULL) { struct ibv_exp_destroy_res_domain_attr attr = { .comp_mask = 0, @@ -780,12 +783,11 @@ rxq_cleanup(struct rxq_ctrl *rxq_ctrl) assert(rxq_ctrl->priv != NULL); assert(rxq_ctrl->priv->ctx != NULL); - claim_zero(ibv_exp_destroy_res_domain(rxq_ctrl->priv->ctx, - rxq_ctrl->rd, - &attr)); + claim_zero(mlx5_glue->exp_destroy_res_domain + (rxq_ctrl->priv->ctx, rxq_ctrl->rd, &attr)); } if (rxq_ctrl->mr != NULL) - claim_zero(ibv_dereg_mr(rxq_ctrl->mr)); + claim_zero(mlx5_glue->dereg_mr(rxq_ctrl->mr)); memset(rxq_ctrl, 0, sizeof(*rxq_ctrl)); } @@ -821,7 +823,7 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl) .attr_mask = IBV_EXP_WQ_ATTR_STATE, .wq_state = IBV_EXP_WQS_RESET, }; - err = ibv_exp_modify_wq(rxq_ctrl->wq, &mod); + err = mlx5_glue->exp_modify_wq(rxq_ctrl->wq, &mod); if (err) { ERROR("%p: cannot reset WQ: %s", (void *)dev, strerror(err)); assert(err > 0); @@ -840,7 +842,7 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl) .attr_mask = IBV_EXP_WQ_ATTR_STATE, .wq_state = IBV_EXP_WQS_RDY, }; - err = ibv_exp_modify_wq(rxq_ctrl->wq, &mod); + err = mlx5_glue->exp_modify_wq(rxq_ctrl->wq, &mod); if (err) { ERROR("%p: WQ state to IBV_EXP_WQS_RDY failed: %s", (void *)dev, strerror(err)); @@ -1007,7 +1009,7 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, .thread_model = IBV_EXP_THREAD_SINGLE, .msg_model = IBV_EXP_MSG_HIGH_BW, }; - tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); + tmpl.rd = mlx5_glue->exp_create_res_domain(priv->ctx, &attr.rd); if (tmpl.rd == NULL) { ret = ENOMEM; ERROR("%p: RD creation failure: %s", @@ -1023,8 +1025,8 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, attr.cq.flags |= IBV_EXP_CQ_COMPRESSED_CQE; cqe_n = (desc * 2) - 1; /* Double the number of CQEs. */ } - tmpl.cq = ibv_exp_create_cq(priv->ctx, cqe_n, NULL, NULL, 0, - &attr.cq); + tmpl.cq = mlx5_glue->exp_create_cq(priv->ctx, cqe_n, NULL, NULL, 0, + &attr.cq); if (tmpl.cq == NULL) { ret = ENOMEM; ERROR("%p: CQ creation failure: %s", @@ -1089,7 +1091,7 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, " up to date", (void *)dev); - tmpl.wq = ibv_exp_create_wq(priv->ctx, &attr.wq); + tmpl.wq = mlx5_glue->exp_create_wq(priv->ctx, &attr.wq); if (tmpl.wq == NULL) { ret = (errno ? errno : EINVAL); ERROR("%p: WQ creation failure: %s", @@ -1118,7 +1120,8 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, .intf = IBV_EXP_INTF_CQ, .obj = tmpl.cq, }; - tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); + tmpl.if_cq = mlx5_glue->exp_query_intf(priv->ctx, &attr.params, + &status); if (tmpl.if_cq == NULL) { ERROR("%p: CQ interface family query failed with status %d", (void *)dev, status); @@ -1129,7 +1132,8 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, .intf = IBV_EXP_INTF_WQ, .obj = tmpl.wq, }; - tmpl.if_wq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); + tmpl.if_wq = mlx5_glue->exp_query_intf(priv->ctx, &attr.params, + &status); if (tmpl.if_wq == NULL) { ERROR("%p: WQ interface family query failed with status %d", (void *)dev, status); @@ -1140,7 +1144,7 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, .attr_mask = IBV_EXP_WQ_ATTR_STATE, .wq_state = IBV_EXP_WQS_RDY, }; - ret = ibv_exp_modify_wq(tmpl.wq, &mod); + ret = mlx5_glue->exp_modify_wq(tmpl.wq, &mod); if (ret) { ERROR("%p: WQ state to IBV_EXP_WQS_RDY failed: %s", (void *)dev, strerror(ret)); diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c index 949035bd47..f85a59d19d 100644 --- a/drivers/net/mlx5/mlx5_txq.c +++ b/drivers/net/mlx5/mlx5_txq.c @@ -62,6 +62,7 @@ #include "mlx5_utils.h" #include "mlx5_defs.h" #include "mlx5.h" +#include "mlx5_glue.h" #include "mlx5_rxtx.h" #include "mlx5_autoconf.h" #include "mlx5_defs.h" @@ -152,9 +153,9 @@ txq_cleanup(struct txq_ctrl *txq_ctrl) params = (struct ibv_exp_release_intf_params){ .comp_mask = 0, }; - claim_zero(ibv_exp_release_intf(txq_ctrl->priv->ctx, - txq_ctrl->if_qp, - ¶ms)); + claim_zero(mlx5_glue->exp_release_intf(txq_ctrl->priv->ctx, + txq_ctrl->if_qp, + ¶ms)); } if (txq_ctrl->if_cq != NULL) { assert(txq_ctrl->priv != NULL); @@ -163,14 +164,14 @@ txq_cleanup(struct txq_ctrl *txq_ctrl) params = (struct ibv_exp_release_intf_params){ .comp_mask = 0, }; - claim_zero(ibv_exp_release_intf(txq_ctrl->priv->ctx, - txq_ctrl->if_cq, - ¶ms)); + claim_zero(mlx5_glue->exp_release_intf(txq_ctrl->priv->ctx, + txq_ctrl->if_cq, + ¶ms)); } if (txq_ctrl->qp != NULL) - claim_zero(ibv_destroy_qp(txq_ctrl->qp)); + claim_zero(mlx5_glue->destroy_qp(txq_ctrl->qp)); if (txq_ctrl->cq != NULL) - claim_zero(ibv_destroy_cq(txq_ctrl->cq)); + claim_zero(mlx5_glue->destroy_cq(txq_ctrl->cq)); if (txq_ctrl->rd != NULL) { struct ibv_exp_destroy_res_domain_attr attr = { .comp_mask = 0, @@ -178,15 +179,14 @@ txq_cleanup(struct txq_ctrl *txq_ctrl) assert(txq_ctrl->priv != NULL); assert(txq_ctrl->priv->ctx != NULL); - claim_zero(ibv_exp_destroy_res_domain(txq_ctrl->priv->ctx, - txq_ctrl->rd, - &attr)); + claim_zero(mlx5_glue->exp_destroy_res_domain + (txq_ctrl->priv->ctx, txq_ctrl->rd, &attr)); } for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) { if (txq_ctrl->txq.mp2mr[i].mp == NULL) break; assert(txq_ctrl->txq.mp2mr[i].mr != NULL); - claim_zero(ibv_dereg_mr(txq_ctrl->txq.mp2mr[i].mr)); + claim_zero(mlx5_glue->dereg_mr(txq_ctrl->txq.mp2mr[i].mr)); } memset(txq_ctrl, 0, sizeof(*txq_ctrl)); } @@ -283,7 +283,7 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, .thread_model = IBV_EXP_THREAD_SINGLE, .msg_model = IBV_EXP_MSG_HIGH_BW, }; - tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); + tmpl.rd = mlx5_glue->exp_create_res_domain(priv->ctx, &attr.rd); if (tmpl.rd == NULL) { ret = ENOMEM; ERROR("%p: RD creation failure: %s", @@ -294,10 +294,11 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, .res_domain = tmpl.rd, }; - tmpl.cq = ibv_exp_create_cq(priv->ctx, - (((desc / MLX5_TX_COMP_THRESH) - 1) ? - ((desc / MLX5_TX_COMP_THRESH) - 1) : 1), - NULL, NULL, 0, &attr.cq); + tmpl.cq = mlx5_glue->exp_create_cq + (priv->ctx, + (((desc / MLX5_TX_COMP_THRESH) - 1) ? + ((desc / MLX5_TX_COMP_THRESH) - 1) : 1), + NULL, NULL, 0, &attr.cq); if (tmpl.cq == NULL) { ret = ENOMEM; ERROR("%p: CQ creation failure: %s", @@ -343,7 +344,7 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, attr.init.cap.max_inline_data = tmpl.txq.max_inline * RTE_CACHE_LINE_SIZE; } - tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init); + tmpl.qp = mlx5_glue->exp_create_qp(priv->ctx, &attr.init); if (tmpl.qp == NULL) { ret = (errno ? errno : EINVAL); ERROR("%p: QP creation failure: %s", @@ -361,8 +362,8 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, /* Primary port number. */ .port_num = priv->port }; - ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, - (IBV_EXP_QP_STATE | IBV_EXP_QP_PORT)); + ret = mlx5_glue->exp_modify_qp(tmpl.qp, &attr.mod, + (IBV_EXP_QP_STATE | IBV_EXP_QP_PORT)); if (ret) { ERROR("%p: QP state to IBV_QPS_INIT failed: %s", (void *)dev, strerror(ret)); @@ -378,14 +379,14 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, attr.mod = (struct ibv_exp_qp_attr){ .qp_state = IBV_QPS_RTR }; - ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE); + ret = mlx5_glue->exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE); if (ret) { ERROR("%p: QP state to IBV_QPS_RTR failed: %s", (void *)dev, strerror(ret)); goto error; } attr.mod.qp_state = IBV_QPS_RTS; - ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE); + ret = mlx5_glue->exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE); if (ret) { ERROR("%p: QP state to IBV_QPS_RTS failed: %s", (void *)dev, strerror(ret)); @@ -396,7 +397,8 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, .intf = IBV_EXP_INTF_CQ, .obj = tmpl.cq, }; - tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); + tmpl.if_cq = mlx5_glue->exp_query_intf(priv->ctx, &attr.params, + &status); if (tmpl.if_cq == NULL) { ret = EINVAL; ERROR("%p: CQ interface family query failed with status %d", @@ -414,7 +416,8 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, IBV_EXP_QP_BURST_CREATE_ENABLE_MULTI_PACKET_SEND_WR : 0), }; - tmpl.if_qp = ibv_exp_query_intf(priv->ctx, &attr.params, &status); + tmpl.if_qp = mlx5_glue->exp_query_intf(priv->ctx, &attr.params, + &status); if (tmpl.if_qp == NULL) { ret = EINVAL; ERROR("%p: QP interface family query failed with status %d", diff --git a/drivers/net/mlx5/mlx5_vlan.c b/drivers/net/mlx5/mlx5_vlan.c index 1b0fa40ac0..0b7e613672 100644 --- a/drivers/net/mlx5/mlx5_vlan.c +++ b/drivers/net/mlx5/mlx5_vlan.c @@ -49,6 +49,7 @@ #include "mlx5_utils.h" #include "mlx5.h" #include "mlx5_autoconf.h" +#include "mlx5_glue.h" /** * Configure a VLAN filter. @@ -159,7 +160,7 @@ priv_vlan_strip_queue_set(struct priv *priv, uint16_t idx, int on) .vlan_offloads = vlan_offloads, }; - err = ibv_exp_modify_wq(rxq_ctrl->wq, &mod); + err = mlx5_glue->exp_modify_wq(rxq_ctrl->wq, &mod); if (err) { ERROR("%p: failed to modified stripping mode: %s", (void *)priv, strerror(err)); diff --git a/mk/rte.app.mk b/mk/rte.app.mk index 92f3635564..bb4456fbd4 100644 --- a/mk/rte.app.mk +++ b/mk/rte.app.mk @@ -114,8 +114,16 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += -lrte_pmd_enic _LDLIBS-$(CONFIG_RTE_LIBRTE_FM10K_PMD) += -lrte_pmd_fm10k _LDLIBS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += -lrte_pmd_i40e _LDLIBS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += -lrte_pmd_ixgbe +ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS),y) +_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += -lrte_pmd_mlx4 -ldl +else _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += -lrte_pmd_mlx4 -libverbs +endif +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y) +_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -ldl +else _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -libverbs +endif _LDLIBS-$(CONFIG_RTE_LIBRTE_MPIPE_PMD) += -lrte_pmd_mpipe -lgxio _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += -lrte_pmd_nfp -lm _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += -lrte_pmd_null