Triggering RDMA WRITE using the doorbell fails

[jinbooooom]

The write function I wrote is as follows:

int xwrite(uint8_t *sq, uint32_t *sqi, uint32_t *qp_dbr,
uint32_t lkey, void *src, uint32_t rkey, void *dst, size_t size,
void *uar_ptr, uint32_t qp) {
struct mlx5_wqe_ctrl_seg *ctrl = (struct mlx5_wqe_ctrl_seg *)(sq + *sqi % SQ_SIZE * MLX5_SEND_WQE_BB);

mlx5dv_set_ctrl_seg(ctrl, *sqi, MLX5_OPCODE_RDMA_WRITE, 0,
		    qp, MLX5_WQE_CTRL_CQ_UPDATE,
		    3, 0, 0);// 控制段(1) + 远程地址段(1) + 数据段(1) = 3个段

struct mlx5_wqe_raddr_seg *rseg = (struct mlx5_wqe_raddr_seg *)(ctrl + 1);
rseg->raddr = htobe64((uintptr_t)dst);
rseg->rkey = htobe32(rkey);
struct mlx5_wqe_data_seg *dseg = (struct mlx5_wqe_data_seg *)(rseg + 1);
mlx5dv_set_data_seg(dseg, size, lkey, (intptr_t)src);
(*sqi)++;
asm volatile("" ::: "memory");
qp_dbr[MLX5_SND_DBR] = htobe32(*sqi & 0xffff);
asm volatile("" ::: "memory");
*(uint64_t *)((uint8_t *)uar_ptr + 0x800) = *(uint64_t *)ctrl;
asm volatile("" ::: "memory");
return 0;

}

[jinbooooom]

In tests/test.cc, the functions recv/xmit are capable of handling reception and transmission. As I aim to utilize the RDMA WRITE functionality, I've written a write function by imitating the xmit function. However, after running it, the communication fails to work. Below are the attempts I've made to tackle this issue:

• I've examined the formats of the control segment and data segment. According to the Mellanox documentation and the code of rdma-core, they seem to be correct.
• Currently, the way the write function triggers the doorbell is the same as that of xmit(send). Is there a difference in the doorbell triggering methods between write and xmit(send)?
• In the write function, I've verified that both the remote address (dst) and the remote key (rkey) are accurate.
• The network cards in use are all mlx5, which support the DEVX extension. The devices can be located using devx_get_device_list.
• Since the recv/xmit functions work properly, it indicates that the environment is fine.

Are there any specific configurations or settings needed for RDMA WRITE? Could any experts help me figure out what's going wrong?
The write function I wrote is as follows:

int xwrite(uint8_t *sq, uint32_t *sqi, uint32_t *qp_dbr,
	   uint32_t lkey, void *src, uint32_t rkey, void *dst, size_t size,
	   void *uar_ptr, uint32_t qp) {
	struct mlx5_wqe_ctrl_seg *ctrl = (struct mlx5_wqe_ctrl_seg *)(sq + *sqi % SQ_SIZE * MLX5_SEND_WQE_BB);
	
	mlx5dv_set_ctrl_seg(ctrl, *sqi, MLX5_OPCODE_RDMA_WRITE, 0,
			    qp, MLX5_WQE_CTRL_CQ_UPDATE,
			    3, 0, 0);// 控制段(1) + 远程地址段(1) + 数据段(1) = 3个段

	struct mlx5_wqe_raddr_seg *rseg = (struct mlx5_wqe_raddr_seg *)(ctrl + 1);
	rseg->raddr = htobe64((uintptr_t)dst);
	rseg->rkey = htobe32(rkey);
	struct mlx5_wqe_data_seg *dseg = (struct mlx5_wqe_data_seg *)(rseg + 1);
	mlx5dv_set_data_seg(dseg, size, lkey, (intptr_t)src);
	(*sqi)++;
	asm volatile("" ::: "memory");
	qp_dbr[MLX5_SND_DBR] = htobe32(*sqi & 0xffff);
	asm volatile("" ::: "memory");
	*(uint64_t *)((uint8_t *)uar_ptr + 0x800) = *(uint64_t *)ctrl;
	asm volatile("" ::: "memory");
	return 0;
}

The original recv/xmit in tests/test.cc is as follows:

int recv(uint8_t *rq, uint32_t *rqi, uint32_t *qp_dbr,
	 uint32_t mkey, void *addr, size_t size) {
	struct mlx5_wqe_data_seg *dseg = (struct mlx5_wqe_data_seg *)(rq + *rqi % RQ_SIZE * MLX5_SEND_WQE_BB);
	mlx5dv_set_data_seg(dseg, size, mkey, (intptr_t)addr);
	// 第二个段通过 MLX5_INVALID_LKEY 表示结束标记
	mlx5dv_set_data_seg(dseg + 1, 0, MLX5_INVALID_LKEY, 0);
	(*rqi)++;
	asm volatile("" ::: "memory");
	qp_dbr[MLX5_RCV_DBR] = htobe32(*rqi & 0xffff);
	return 0;
}

int xmit(uint8_t *sq, uint32_t *sqi, uint32_t *qp_dbr,
	 uint32_t mkey, void *addr, size_t size,
	 void *uar_ptr, uint32_t qp) {

	// 准备WQE控制段和数据段
	struct mlx5_wqe_ctrl_seg *ctrl = (struct mlx5_wqe_ctrl_seg *)(sq + *sqi % SQ_SIZE * MLX5_SEND_WQE_BB);
	mlx5dv_set_ctrl_seg(ctrl, *sqi, MLX5_OPCODE_SEND, 0,
			    qp, MLX5_WQE_CTRL_CQ_UPDATE,
			    2, 0, 0); // ds=2，因为控制段(1) + 数据段(1) = 2个段
	struct mlx5_wqe_data_seg *dseg = (struct mlx5_wqe_data_seg *)(ctrl + 1);
	mlx5dv_set_data_seg(dseg, size, mkey, (intptr_t)addr);

	(*sqi)++;
	asm volatile("" ::: "memory");
	qp_dbr[MLX5_SND_DBR] = htobe32(*sqi & 0xffff);
	asm volatile("" ::: "memory");

	*(uint64_t *)((uint8_t *)uar_ptr + 0x800) = *(uint64_t *)ctrl;
	asm volatile("" ::: "memory");
	return 0;
}