From c2dcc4b05b0507f474bd8454de07160b5c475c5b Mon Sep 17 00:00:00 2001
From: Maurizio Lombardi <mlombard@redhat.com>
Date: Fri, 7 Feb 2025 15:37:25 +0100
Subject: [PATCH 01/12] nvme: add definitions for cancel command

Add new definitions needed to support TP4097a.

Signed-off-by: John Meneghini <jmeneghi@redhat.com>
Signed-off-by: Maurizio Lombardi <mlombard@redhat.com>
---
 drivers/nvme/host/constants.c |  1 +
 drivers/nvme/host/core.c      |  1 +
 include/linux/nvme.h          | 22 ++++++++++++++++++++++
 3 files changed, 24 insertions(+)

diff --git a/drivers/nvme/host/constants.c b/drivers/nvme/host/constants.c
index 2b9e6cfaf2a80a..ff6e965b372d39 100644
--- a/drivers/nvme/host/constants.c
+++ b/drivers/nvme/host/constants.c
@@ -19,6 +19,7 @@ static const char * const nvme_ops[] = {
 	[nvme_cmd_resv_report] = "Reservation Report",
 	[nvme_cmd_resv_acquire] = "Reservation Acquire",
 	[nvme_cmd_resv_release] = "Reservation Release",
+	[nvme_cmd_cancel] = "Cancel",
 	[nvme_cmd_zone_mgmt_send] = "Zone Management Send",
 	[nvme_cmd_zone_mgmt_recv] = "Zone Management Receive",
 	[nvme_cmd_zone_append] = "Zone Append",
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 870314c521077c..95aeace93b75f1 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -5087,6 +5087,7 @@ static inline void _nvme_check_size(void)
 	BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
 	BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
 	BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
+	BUILD_BUG_ON(sizeof(struct nvme_cancel_cmd) != 64);
 	BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
 	BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
 	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
diff --git a/include/linux/nvme.h b/include/linux/nvme.h
index bfb5688363b0dc..cdf22a859f9b8f 100644
--- a/include/linux/nvme.h
+++ b/include/linux/nvme.h
@@ -28,6 +28,9 @@
 /* Special NSSR value, 'NVMe' */
 #define NVME_SUBSYS_RESET	0x4E564D65
 
+/* Maximum number of reserved commands for Cancel */
+#define NVME_RSV_CANCEL_MAX     2
+
 enum nvme_subsys_type {
 	/* Referral to another discovery type target subsystem */
 	NVME_NQN_DISC	= 1,
@@ -890,6 +893,7 @@ enum nvme_opcode {
 	nvme_cmd_resv_report	= 0x0e,
 	nvme_cmd_resv_acquire	= 0x11,
 	nvme_cmd_resv_release	= 0x15,
+	nvme_cmd_cancel		= 0x18,
 	nvme_cmd_zone_mgmt_send	= 0x79,
 	nvme_cmd_zone_mgmt_recv	= 0x7a,
 	nvme_cmd_zone_append	= 0x7d,
@@ -1425,6 +1429,22 @@ struct nvme_abort_cmd {
 	__u32			rsvd11[5];
 };
 
+struct nvme_cancel_cmd {
+	__u8			opcode;
+	__u8			flags;
+	__u16			command_id;
+	__le32			nsid;
+	__u32                   rsvd1[8];
+	__le16			sqid;
+	__u16			cid;
+	__u8			action;
+	__u8			rsvd11[3];
+	__u32			rsvd12[4];
+};
+
+#define NVME_CANCEL_ACTION_MUL_CMD	1
+#define NVME_CANCEL_ACTION_SINGLE_CMD	0
+
 struct nvme_download_firmware {
 	__u8			opcode;
 	__u8			flags;
@@ -1887,6 +1907,7 @@ struct nvme_command {
 		struct nvme_zone_mgmt_send_cmd zms;
 		struct nvme_zone_mgmt_recv_cmd zmr;
 		struct nvme_abort_cmd abort;
+		struct nvme_cancel_cmd cancel;
 		struct nvme_get_log_page_command get_log_page;
 		struct nvmf_common_command fabrics;
 		struct nvmf_connect_command connect;
@@ -2069,6 +2090,7 @@ enum {
 	NVME_SC_INVALID_PI		= 0x181,
 	NVME_SC_READ_ONLY		= 0x182,
 	NVME_SC_ONCS_NOT_SUPPORTED	= 0x183,
+	NVME_SC_INVALID_CID		= 0x184,
 
 	/*
 	 * I/O Command Set Specific - Fabrics commands:

From 59f4577f2e4c0123750c216ffa44dc5c469fddc2 Mon Sep 17 00:00:00 2001
From: Maurizio Lombardi <mlombard@redhat.com>
Date: Fri, 7 Feb 2025 15:37:30 +0100
Subject: [PATCH 02/12] nvme-core: reserve tags for cancel commands

If the controller supports cancel commands, reserve
NVME_RSV_CANCEL_MAX tags for them.

Signed-off-by: Maurizio Lombardi <mlombard@redhat.com>
---
 drivers/nvme/host/core.c | 15 ++++++++++++++-
 drivers/nvme/host/nvme.h |  1 +
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 95aeace93b75f1..f5e6d7cd152a25 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -1210,6 +1210,14 @@ u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
 }
 EXPORT_SYMBOL_NS_GPL(nvme_command_effects, "NVME_TARGET_PASSTHRU");
 
+bool nvme_io_command_supported(struct nvme_ctrl *ctrl, u8 opcode)
+{
+	u32 effects = le32_to_cpu(ctrl->effects->iocs[opcode]);
+
+	return effects & NVME_CMD_EFFECTS_CSUPP;
+}
+EXPORT_SYMBOL_GPL(nvme_io_command_supported);
+
 u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
 {
 	u32 effects = nvme_command_effects(ctrl, ns, opcode);
@@ -4675,9 +4683,14 @@ int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
 	 */
 	if (ctrl->quirks & NVME_QUIRK_SHARED_TAGS)
 		set->reserved_tags = NVME_AQ_DEPTH;
-	else if (ctrl->ops->flags & NVME_F_FABRICS)
+	else if (ctrl->ops->flags & NVME_F_FABRICS) {
 		/* Reserved for fabric connect */
 		set->reserved_tags = 1;
+		if (nvme_io_command_supported(ctrl, nvme_cmd_cancel)) {
+			/* Reserved for cancel commands */
+			set->reserved_tags += NVME_RSV_CANCEL_MAX;
+		}
+	}
 	set->numa_node = ctrl->numa_node;
 	if (ctrl->ops->flags & NVME_F_BLOCKING)
 		set->flags |= BLK_MQ_F_BLOCKING;
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 51e07864212710..6dfc875ddfd172 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -1187,6 +1187,7 @@ static inline void nvme_auth_revoke_tls_key(struct nvme_ctrl *ctrl) {};
 
 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 			 u8 opcode);
+bool nvme_io_command_supported(struct nvme_ctrl *ctrl, u8 opcode);
 u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode);
 int nvme_execute_rq(struct request *rq, bool at_head);
 void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects,

From b5820117c17b508bbfb2ea53466d9bc9a431841b Mon Sep 17 00:00:00 2001
From: Maurizio Lombardi <mlombard@redhat.com>
Date: Fri, 7 Feb 2025 15:37:39 +0100
Subject: [PATCH 03/12] nvme-core: add a function to submit a cancel command

Add a function to send a cancel command to abort the specified request.
When the cancel command completes, the host driver will print the
number of deferred and immediate aborts performed by the target.

Signed-off-by: Maurizio Lombardi <mlombard@redhat.com>
---
 drivers/nvme/host/core.c | 61 ++++++++++++++++++++++++++++++++++++++++
 drivers/nvme/host/nvme.h |  2 ++
 2 files changed, 63 insertions(+)

diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index f5e6d7cd152a25..d0dbc249b575bb 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -3113,6 +3113,67 @@ int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
 	return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
 }
 
+static enum rq_end_io_ret nvme_cancel_endio(struct request *req, blk_status_t error)
+{
+	struct nvme_ctrl *ctrl = req->end_io_data;
+	u32 result;
+	u16 imm_abrts, def_abrts;
+
+	if (!error) {
+		result = le32_to_cpu(nvme_req(req)->result.u32);
+		def_abrts = upper_16_bits(result);
+		imm_abrts = lower_16_bits(result);
+
+		dev_warn(ctrl->device,
+			 "Cancel status: 0x0 imm abrts = %u def abrts = %u",
+			 imm_abrts, def_abrts);
+	} else {
+		dev_warn(ctrl->device, "Cancel status: 0x%x",
+			 nvme_req(req)->status);
+	}
+
+	blk_mq_free_request(req);
+	return RQ_END_IO_NONE;
+}
+
+int nvme_submit_cancel_req(struct nvme_ctrl *ctrl, struct request *rq,
+			   unsigned int sqid, int action)
+{
+	struct nvme_command c = { };
+	struct request *cancel_req;
+
+	if (sqid == 0)
+		return -EINVAL;
+
+	c.cancel.opcode = nvme_cmd_cancel;
+	c.cancel.sqid = cpu_to_le32(sqid);
+	c.cancel.nsid = NVME_NSID_ALL;
+	c.cancel.action = action;
+	if (action == NVME_CANCEL_ACTION_SINGLE_CMD)
+		c.cancel.cid = nvme_cid(rq);
+	else
+		c.cancel.cid = 0xFFFF;
+
+	cancel_req = blk_mq_alloc_request_hctx(rq->q, nvme_req_op(&c),
+					       BLK_MQ_REQ_NOWAIT |
+					       BLK_MQ_REQ_RESERVED,
+					       sqid - 1);
+	if (IS_ERR(cancel_req)) {
+		dev_warn(ctrl->device, "%s: Could not allocate the Cancel "
+					"command", __func__);
+		return PTR_ERR(cancel_req);
+	}
+
+	nvme_init_request(cancel_req, &c);
+	cancel_req->end_io = nvme_cancel_endio;
+	cancel_req->end_io_data = ctrl;
+
+	blk_execute_rq_nowait(cancel_req, false);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nvme_submit_cancel_req);
+
 static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
 				struct nvme_effects_log **log)
 {
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 6dfc875ddfd172..6388f0a52638a2 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -908,6 +908,8 @@ int nvme_delete_ctrl(struct nvme_ctrl *ctrl);
 void nvme_queue_scan(struct nvme_ctrl *ctrl);
 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
 		void *log, size_t size, u64 offset);
+int nvme_submit_cancel_req(struct nvme_ctrl *ctrl, struct request *rq,
+			       unsigned int sqid, int action);
 bool nvme_tryget_ns_head(struct nvme_ns_head *head);
 void nvme_put_ns_head(struct nvme_ns_head *head);
 int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device,

From 4f98da759af6be444bfc195588c19a63487738ab Mon Sep 17 00:00:00 2001
From: Maurizio Lombardi <mlombard@redhat.com>
Date: Thu, 20 Mar 2025 15:56:51 +0100
Subject: [PATCH 04/12] nvme-tcp: use the cancel command to perform an abort if
 target supports it

If available, use Cancel command to abort the command that timed
out instead of resetting the controller.
Limit the number of outstanding cancel commands
to a maximum of 2 per queue;
If more than one command time out at the same time, the driver will use a
Cancel command with action flag set to "Multiple commands" to abort all
the commands on the specified queue as a last resort to avoid a
controller reset.

If the cancel command is not supported or if any error is encountered,
the driver will fall back to the normal controller reset.

Signed-off-by: Maurizio Lombardi <mlombard@redhat.com>
---
 drivers/nvme/host/core.c |  6 ++++
 drivers/nvme/host/nvme.h |  1 +
 drivers/nvme/host/tcp.c  | 62 ++++++++++++++++++++++++++++++++++++----
 3 files changed, 64 insertions(+), 5 deletions(-)

diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index d0dbc249b575bb..ad141028bcffe6 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -1218,6 +1218,12 @@ bool nvme_io_command_supported(struct nvme_ctrl *ctrl, u8 opcode)
 }
 EXPORT_SYMBOL_GPL(nvme_io_command_supported);
 
+bool nvme_is_cancel(struct nvme_command *cmd)
+{
+	return cmd->common.opcode == nvme_cmd_cancel;
+}
+EXPORT_SYMBOL_GPL(nvme_is_cancel);
+
 u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
 {
 	u32 effects = nvme_command_effects(ctrl, ns, opcode);
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 6388f0a52638a2..1225a6ff4093e8 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -1190,6 +1190,7 @@ static inline void nvme_auth_revoke_tls_key(struct nvme_ctrl *ctrl) {};
 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 			 u8 opcode);
 bool nvme_io_command_supported(struct nvme_ctrl *ctrl, u8 opcode);
+bool nvme_is_cancel(struct nvme_command *cmd);
 u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode);
 int nvme_execute_rq(struct request *rq, bool at_head);
 void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects,
diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index 542ffc921a3ff8..39de42ae3a7e96 100644
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -122,6 +122,7 @@ struct nvme_tcp_request {
 	size_t			offset;
 	size_t			data_sent;
 	enum nvme_tcp_send_state state;
+	bool			aborted;
 };
 
 enum nvme_tcp_queue_flags {
@@ -129,6 +130,8 @@ enum nvme_tcp_queue_flags {
 	NVME_TCP_Q_LIVE		= 1,
 	NVME_TCP_Q_POLLING	= 2,
 	NVME_TCP_Q_IO_CPU_SET	= 3,
+	NVME_TCP_Q_CANCEL_ONE   = 4,
+	NVME_TCP_Q_CANCEL_ALL   = 5,
 };
 
 enum nvme_tcp_recv_state {
@@ -205,6 +208,7 @@ static struct workqueue_struct *nvme_tcp_wq;
 static const struct blk_mq_ops nvme_tcp_mq_ops;
 static const struct blk_mq_ops nvme_tcp_admin_mq_ops;
 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue);
+static void nvme_tcp_complete_rq(struct request *rq);
 
 static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl)
 {
@@ -617,7 +621,7 @@ static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,
 		req->status = cqe->status;
 
 	if (!nvme_try_complete_req(rq, req->status, cqe->result))
-		nvme_complete_rq(rq);
+		nvme_tcp_complete_rq(rq);
 	queue->nr_cqe++;
 
 	return 0;
@@ -817,7 +821,7 @@ static inline void nvme_tcp_end_request(struct request *rq, u16 status)
 	union nvme_result res = {};
 
 	if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res))
-		nvme_complete_rq(rq);
+		nvme_tcp_complete_rq(rq);
 }
 
 static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
@@ -1833,6 +1837,8 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
 		goto err_init_connect;
 
 	set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags);
+	clear_bit(NVME_TCP_Q_CANCEL_ONE, &queue->flags);
+	clear_bit(NVME_TCP_Q_CANCEL_ALL, &queue->flags);
 
 	return 0;
 
@@ -2551,10 +2557,12 @@ static void nvme_tcp_complete_timed_out(struct request *rq)
 static enum blk_eh_timer_return nvme_tcp_timeout(struct request *rq)
 {
 	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
-	struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
 	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
 	struct nvme_command *cmd = &pdu->cmd;
-	int qid = nvme_tcp_queue_id(req->queue);
+	struct nvme_tcp_queue *queue = req->queue;
+	struct nvme_ctrl *ctrl = &queue->ctrl->ctrl;
+	int qid = nvme_tcp_queue_id(queue);
+	int error, action;
 
 	dev_warn(ctrl->device,
 		 "I/O tag %d (%04x) type %d opcode %#x (%s) QID %d timeout\n",
@@ -2579,10 +2587,39 @@ static enum blk_eh_timer_return nvme_tcp_timeout(struct request *rq)
 		return BLK_EH_DONE;
 	}
 
+	if (!req->aborted) {
+		if (!nvme_io_command_supported(ctrl, nvme_cmd_cancel) || !qid)
+			goto err_recovery;
+
+		if (!test_and_set_bit(NVME_TCP_Q_CANCEL_ONE, &queue->flags)) {
+			action = NVME_CANCEL_ACTION_SINGLE_CMD;
+		} else if (!test_and_set_bit(NVME_TCP_Q_CANCEL_ALL,
+						&queue->flags)) {
+			action = NVME_CANCEL_ACTION_MUL_CMD;
+		} else {
+			/* No free reserved commands.
+			 * this means a "multiple commands" cancel
+			 * is currently under execution and this request
+			 * is likely to be canceled. Mark this
+			 * request as aborted and reset the timer.
+			 */
+			goto abort;
+		}
+
+		error = nvme_submit_cancel_req(ctrl, rq, qid, action);
+		if (error)
+			goto err_recovery;
+
+abort:
+		req->aborted = true;
+		return BLK_EH_RESET_TIMER;
+	}
+
 	/*
 	 * LIVE state should trigger the normal error recovery which will
 	 * handle completing this request.
 	 */
+err_recovery:
 	nvme_tcp_error_recovery(ctrl);
 	return BLK_EH_RESET_TIMER;
 }
@@ -2627,6 +2664,7 @@ static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns,
 	req->pdu_len = 0;
 	req->pdu_sent = 0;
 	req->h2cdata_left = 0;
+	req->aborted = false;
 	req->data_len = blk_rq_nr_phys_segments(rq) ?
 				blk_rq_payload_bytes(rq) : 0;
 	req->curr_bio = rq->bio;
@@ -2742,10 +2780,24 @@ static int nvme_tcp_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
 	return len;
 }
 
+static void nvme_tcp_complete_rq(struct request *rq)
+{
+	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
+	struct nvme_tcp_queue *queue = req->queue;
+	bool is_cancel = nvme_is_cancel(req->req.cmd);
+
+	nvme_complete_rq(rq);
+
+	if (is_cancel) {
+		if (!test_and_clear_bit(NVME_TCP_Q_CANCEL_ALL, &queue->flags))
+			clear_bit(NVME_TCP_Q_CANCEL_ONE, &queue->flags);
+	}
+}
+
 static const struct blk_mq_ops nvme_tcp_mq_ops = {
 	.queue_rq	= nvme_tcp_queue_rq,
 	.commit_rqs	= nvme_tcp_commit_rqs,
-	.complete	= nvme_complete_rq,
+	.complete	= nvme_tcp_complete_rq,
 	.init_request	= nvme_tcp_init_request,
 	.exit_request	= nvme_tcp_exit_request,
 	.init_hctx	= nvme_tcp_init_hctx,

From 87994fd59b00e97c4f261de319aca77e575e9e66 Mon Sep 17 00:00:00 2001
From: Maurizio Lombardi <mlombard@redhat.com>
Date: Fri, 7 Feb 2025 15:40:27 +0100
Subject: [PATCH 05/12] nvme-rdma: use the cancel command to perform an abort
 if target supports it

If available, use Cancel command to abort the command that timed
out instead of resetting the controller.

Limit the number of outstanding cancel commands
to a maximum of 2 per queue;
If more than one command time out at the same time, the driver will use a
Cancel command with action flag set to "Multiple commands" to abort all
the commands on the specified queue as a last resort to avoid a
controller reset.

If the cancel command is not supported or if any error is encountered,
the driver will fall back to the normal controller reset.

Signed-off-by: Maurizio Lombardi <mlombard@redhat.com>
---
 drivers/nvme/host/rdma.c | 50 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 46 insertions(+), 4 deletions(-)

diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index b5a0295b5bf457..3de37d116e4aa9 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -74,12 +74,15 @@ struct nvme_rdma_request {
 	struct nvme_rdma_sgl	data_sgl;
 	struct nvme_rdma_sgl	*metadata_sgl;
 	bool			use_sig_mr;
+	bool			aborted;
 };
 
 enum nvme_rdma_queue_flags {
 	NVME_RDMA_Q_ALLOCATED		= 0,
 	NVME_RDMA_Q_LIVE		= 1,
 	NVME_RDMA_Q_TR_READY		= 2,
+	NVME_RDMA_Q_CANCEL_ONE		= 3,
+	NVME_RDMA_Q_CANCEL_ALL		= 4,
 };
 
 struct nvme_rdma_queue {
@@ -619,6 +622,8 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
 	}
 
 	set_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags);
+	clear_bit(NVME_RDMA_Q_CANCEL_ONE, &queue->flags);
+	clear_bit(NVME_RDMA_Q_CANCEL_ALL, &queue->flags);
 
 	return 0;
 
@@ -1954,16 +1959,18 @@ static enum blk_eh_timer_return nvme_rdma_timeout(struct request *rq)
 {
 	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
 	struct nvme_rdma_queue *queue = req->queue;
-	struct nvme_rdma_ctrl *ctrl = queue->ctrl;
+	struct nvme_rdma_ctrl *rdma_ctrl = queue->ctrl;
+	struct nvme_ctrl *ctrl = &rdma_ctrl->ctrl;
 	struct nvme_command *cmd = req->req.cmd;
 	int qid = nvme_rdma_queue_idx(queue);
+	int error, action;
 
-	dev_warn(ctrl->ctrl.device,
+	dev_warn(ctrl->device,
 		 "I/O tag %d (%04x) opcode %#x (%s) QID %d timeout\n",
 		 rq->tag, nvme_cid(rq), cmd->common.opcode,
 		 nvme_fabrics_opcode_str(qid, cmd), qid);
 
-	if (nvme_ctrl_state(&ctrl->ctrl) != NVME_CTRL_LIVE) {
+	if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) {
 		/*
 		 * If we are resetting, connecting or deleting we should
 		 * complete immediately because we may block controller
@@ -1981,11 +1988,40 @@ static enum blk_eh_timer_return nvme_rdma_timeout(struct request *rq)
 		return BLK_EH_DONE;
 	}
 
+	if (!req->aborted) {
+		if (!nvme_io_command_supported(ctrl, nvme_cmd_cancel) || !qid)
+			goto err_recovery;
+
+		if (!test_and_set_bit(NVME_RDMA_Q_CANCEL_ONE, &queue->flags)) {
+			action = NVME_CANCEL_ACTION_SINGLE_CMD;
+		} else if (!test_and_set_bit(NVME_RDMA_Q_CANCEL_ALL,
+						&queue->flags)) {
+			action = NVME_CANCEL_ACTION_MUL_CMD;
+		} else {
+			/* No free reserved commands.
+			 * this means a "multiple commands" cancel
+			 * is currently under execution and this request
+			 * is likely to be canceled. Mark this
+			 * request as aborted and reset the timer.
+			 */
+			goto abort;
+		}
+
+		error = nvme_submit_cancel_req(ctrl, rq, qid, action);
+		if (error)
+			goto err_recovery;
+
+abort:
+		req->aborted = true;
+		return BLK_EH_RESET_TIMER;
+	}
+
 	/*
 	 * LIVE state should trigger the normal error recovery which will
 	 * handle completing this request.
 	 */
-	nvme_rdma_error_recovery(ctrl);
+err_recovery:
+	nvme_rdma_error_recovery(rdma_ctrl);
 	return BLK_EH_RESET_TIMER;
 }
 
@@ -2009,6 +2045,7 @@ static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
 		return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);
 
 	dev = queue->device->dev;
+	req->aborted = false;
 
 	req->sqe.dma = ib_dma_map_single(dev, req->sqe.data,
 					 sizeof(struct nvme_command),
@@ -2113,6 +2150,7 @@ static void nvme_rdma_complete_rq(struct request *rq)
 	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
 	struct nvme_rdma_queue *queue = req->queue;
 	struct ib_device *ibdev = queue->device->dev;
+	bool is_cancel = nvme_is_cancel(req->req.cmd);
 
 	if (req->use_sig_mr)
 		nvme_rdma_check_pi_status(req);
@@ -2121,6 +2159,10 @@ static void nvme_rdma_complete_rq(struct request *rq)
 	ib_dma_unmap_single(ibdev, req->sqe.dma, sizeof(struct nvme_command),
 			    DMA_TO_DEVICE);
 	nvme_complete_rq(rq);
+	if (is_cancel) {
+		if (!test_and_clear_bit(NVME_RDMA_Q_CANCEL_ALL, &queue->flags))
+			clear_bit(NVME_RDMA_Q_CANCEL_ONE, &queue->flags);
+	}
 }
 
 static void nvme_rdma_map_queues(struct blk_mq_tag_set *set)

From 8d17029513a3fe4f0ba117f4d20f5f0611633b34 Mon Sep 17 00:00:00 2001
From: Maurizio Lombardi <mlombard@redhat.com>
Date: Fri, 7 Feb 2025 15:40:41 +0100
Subject: [PATCH 06/12] nvmet: emulate basic support for cancel commands

Add a module parameter that allows the user to enable a basic
cancel command emulation, this could be useful for testing the host
driver's cancel command implementation.

It just reports that no abort has been executed, it will however
do some basic sanity check on the command sent by the host and return
an error if the fields are invalid.

Signed-off-by: Maurizio Lombardi <mlombard@redhat.com>
---
 drivers/nvme/target/Makefile        |  2 +-
 drivers/nvme/target/admin-cmd.c     |  4 +++
 drivers/nvme/target/core.c          | 11 ++++++
 drivers/nvme/target/io-cmd-cancel.c | 52 +++++++++++++++++++++++++++++
 drivers/nvme/target/nvmet.h         |  4 +++
 5 files changed, 72 insertions(+), 1 deletion(-)
 create mode 100644 drivers/nvme/target/io-cmd-cancel.c

diff --git a/drivers/nvme/target/Makefile b/drivers/nvme/target/Makefile
index ed8522911d1f73..a533357e9be4a7 100644
--- a/drivers/nvme/target/Makefile
+++ b/drivers/nvme/target/Makefile
@@ -11,7 +11,7 @@ obj-$(CONFIG_NVME_TARGET_TCP)		+= nvmet-tcp.o
 obj-$(CONFIG_NVME_TARGET_PCI_EPF)	+= nvmet-pci-epf.o
 
 nvmet-y		+= core.o configfs.o admin-cmd.o fabrics-cmd.o \
-			discovery.o io-cmd-file.o io-cmd-bdev.o pr.o
+			discovery.o io-cmd-file.o io-cmd-bdev.o pr.o io-cmd-cancel.o
 nvmet-$(CONFIG_NVME_TARGET_DEBUGFS)	+= debugfs.o
 nvmet-$(CONFIG_NVME_TARGET_PASSTHRU)	+= passthru.o
 nvmet-$(CONFIG_BLK_DEV_ZONED)		+= zns.o
diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c
index acc138bbf8f23d..57b90263a7cb95 100644
--- a/drivers/nvme/target/admin-cmd.c
+++ b/drivers/nvme/target/admin-cmd.c
@@ -398,6 +398,10 @@ static void nvmet_get_cmd_effects_nvm(struct nvme_effects_log *log)
 	log->iocs[nvme_cmd_resv_release] =
 	log->iocs[nvme_cmd_resv_report] =
 		cpu_to_le32(NVME_CMD_EFFECTS_CSUPP);
+	if (emulate_cancel_support) {
+		log->iocs[nvme_cmd_cancel] =
+			cpu_to_le32(NVME_CMD_EFFECTS_CSUPP);
+	}
 	log->iocs[nvme_cmd_write] =
 	log->iocs[nvme_cmd_write_zeroes] =
 		cpu_to_le32(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC);
diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index a2b5319c37f350..3e22dacddd76a7 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -27,6 +27,10 @@ static DEFINE_IDA(cntlid_ida);
 struct workqueue_struct *nvmet_wq;
 EXPORT_SYMBOL_GPL(nvmet_wq);
 
+bool emulate_cancel_support;
+module_param(emulate_cancel_support, bool, 0644);
+MODULE_PARM_DESC(emulate_cancel_support, "Emulate the cancel command support. Default = false");
+
 /*
  * This read/write semaphore is used to synchronize access to configuration
  * information on a target system that will result in discovery log page
@@ -1057,6 +1061,13 @@ static u16 nvmet_parse_io_cmd(struct nvmet_req *req)
 	if (nvmet_is_passthru_req(req))
 		return nvmet_parse_passthru_io_cmd(req);
 
+	if (emulate_cancel_support &&
+	    req->cmd->common.opcode == nvme_cmd_cancel) {
+		req->execute = nvmet_execute_cancel;
+		if (req->cmd->cancel.nsid == NVME_NSID_ALL)
+			return 0;
+	}
+
 	ret = nvmet_req_find_ns(req);
 	if (unlikely(ret))
 		return ret;
diff --git a/drivers/nvme/target/io-cmd-cancel.c b/drivers/nvme/target/io-cmd-cancel.c
new file mode 100644
index 00000000000000..8f55458e646a25
--- /dev/null
+++ b/drivers/nvme/target/io-cmd-cancel.c
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * NVMe I/O cancel command implementation.
+ * Copyright (c) 2023 Red Hat
+ */
+
+#include "nvmet.h"
+
+void nvmet_execute_cancel(struct nvmet_req *req)
+{
+	u16 cid;
+	__le16 sqid;
+	bool mult_cmds;
+	int ret = 0;
+	struct nvmet_ctrl *ctrl = req->sq->ctrl;
+	struct nvmet_sq *sq = req->sq;
+
+	if (!nvmet_check_transfer_len(req, 0))
+		return;
+
+	sqid = le16_to_cpu(req->cmd->cancel.sqid);
+	if (sqid > ctrl->subsys->max_qid) {
+		ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
+		goto exit;
+	}
+
+	mult_cmds = req->cmd->cancel.action & NVME_CANCEL_ACTION_MUL_CMD;
+	cid  = req->cmd->cancel.cid;
+
+	if (cid == req->cmd->cancel.command_id && !mult_cmds) {
+		/* If action is set to "single command" and cid is
+		 * set to the cid of this cancel command, then
+		 * the controller shall abort the command with
+		 * an "invalid cid" status code.
+		 */
+		ret = NVME_SC_INVALID_CID | NVME_STATUS_DNR;
+	} else if ((cid != 0xFFFF && mult_cmds) || sqid != sq->qid) {
+		/* if action is set to "multiple commands" and
+		 * cid isn't set to 0xFFFF, then abort the command
+		 * with an "invalid field" status.
+		 * if the sqid field doesn't match the sqid of
+		 * the queue to which the cancel command is submitted,
+		 * then abort the command with an "invalid field" status.
+		 */
+		ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
+	}
+
+exit:
+	nvmet_set_result(req, 0);
+	nvmet_req_complete(req, ret);
+}
+
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index 33fac9151b5b14..8778fbe82e740b 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -40,6 +40,8 @@
 #define nvmet_for_each_enabled_ns(xa, index, entry) \
 	xa_for_each_marked(xa, index, entry, NVMET_NS_ENABLED)
 
+extern bool emulate_cancel_support;
+
 /*
  * Supported optional AENs:
  */
@@ -714,6 +716,8 @@ void nvmet_bdev_execute_zone_mgmt_recv(struct nvmet_req *req);
 void nvmet_bdev_execute_zone_mgmt_send(struct nvmet_req *req);
 void nvmet_bdev_execute_zone_append(struct nvmet_req *req);
 
+void nvmet_execute_cancel(struct nvmet_req *req);
+
 static inline u32 nvmet_rw_data_len(struct nvmet_req *req)
 {
 	return ((u32)le16_to_cpu(req->cmd->rw.length) + 1) <<

From 5c968a45e8398c1ebd25ba2c3d860b4b2b0989b3 Mon Sep 17 00:00:00 2001
From: Chris Leech <cleech@redhat.com>
Date: Tue, 14 Jan 2025 13:08:21 -0800
Subject: [PATCH 07/12] nvmet: put all nvmet_req.execute calls behind a
 function name

This is setup for being able to manipluate the outstanding requests.

Signed-off-by: Chris Leech <cleech@redhat.com>
---
 drivers/nvme/target/fc.c    | 4 ++--
 drivers/nvme/target/loop.c  | 2 +-
 drivers/nvme/target/nvmet.h | 1 +
 drivers/nvme/target/rdma.c  | 4 ++--
 drivers/nvme/target/tcp.c   | 4 ++--
 5 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c
index 7318b736d41417..f86b628b1efef7 100644
--- a/drivers/nvme/target/fc.c
+++ b/drivers/nvme/target/fc.c
@@ -2378,7 +2378,7 @@ nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod)
 		}
 
 		/* data transfer complete, resume with nvmet layer */
-		fod->req.execute(&fod->req);
+		nvmet_execute_request(&fod->req);
 		break;
 
 	case NVMET_FCOP_READDATA:
@@ -2589,7 +2589,7 @@ nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
 	 * can invoke the nvmet_layer now. If read data, cmd completion will
 	 * push the data
 	 */
-	fod->req.execute(&fod->req);
+	nvmet_execute_request(&fod->req);
 	return;
 
 transport_error:
diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c
index a5c41144667c68..11d5f6dd9ece51 100644
--- a/drivers/nvme/target/loop.c
+++ b/drivers/nvme/target/loop.c
@@ -125,7 +125,7 @@ static void nvme_loop_execute_work(struct work_struct *work)
 	struct nvme_loop_iod *iod =
 		container_of(work, struct nvme_loop_iod, work);
 
-	iod->req.execute(&iod->req);
+	nvmet_execute_request(&iod->req);
 }
 
 static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index 8778fbe82e740b..7599dbb4347cb2 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -967,4 +967,5 @@ struct nvmet_feat_arbitration {
 	u8		ab;
 };
 
+static inline void nvmet_execute_request(struct nvmet_req *req) { req->execute(req); }
 #endif /* _NVMET_H */
diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c
index 1afd93026f9bf0..f9ffb5feedb5c2 100644
--- a/drivers/nvme/target/rdma.c
+++ b/drivers/nvme/target/rdma.c
@@ -773,7 +773,7 @@ static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc)
 	if (unlikely(status))
 		nvmet_req_complete(&rsp->req, status);
 	else
-		rsp->req.execute(&rsp->req);
+		nvmet_execute_request(&rsp->req);
 }
 
 static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc)
@@ -958,7 +958,7 @@ static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp)
 				queue->cm_id->port_num, &rsp->read_cqe, NULL))
 			nvmet_req_complete(&rsp->req, NVME_SC_DATA_XFER_ERROR);
 	} else {
-		rsp->req.execute(&rsp->req);
+		nvmet_execute_request(&rsp->req);
 	}
 
 	return true;
diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c
index 9cf97433b0b6d5..96851e226bc4ed 100644
--- a/drivers/nvme/target/tcp.c
+++ b/drivers/nvme/target/tcp.c
@@ -597,7 +597,7 @@ static void nvmet_tcp_execute_request(struct nvmet_tcp_cmd *cmd)
 	if (unlikely(cmd->flags & NVMET_TCP_F_INIT_FAILED))
 		nvmet_tcp_queue_response(&cmd->req);
 	else
-		cmd->req.execute(&cmd->req);
+		nvmet_execute_request(&cmd->req);
 }
 
 static int nvmet_try_send_data_pdu(struct nvmet_tcp_cmd *cmd)
@@ -1104,7 +1104,7 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue)
 		goto out;
 	}
 
-	queue->cmd->req.execute(&queue->cmd->req);
+	nvmet_execute_request(&queue->cmd->req);
 out:
 	nvmet_prepare_receive_pdu(queue);
 	return ret;

From c2a13a080bb85b0ca228204c42ce1c209d47df94 Mon Sep 17 00:00:00 2001
From: Chris Leech <cleech@redhat.com>
Date: Tue, 18 Mar 2025 15:38:05 -0700
Subject: [PATCH 08/12] nvmet: add delay debugfs file to nvmet_ctrl

Creates an delay attribute file on the controler in debugfs
/sys/kernel/debug/nvmet/<subsystem>/ctrlN/delay

Reading this file returns two numbers, a reqeust delay count and a delay
time in ms.  Each delayed request will decrement the delay count until
it reaches 0.

Writing to this file can set both the delay and count at once, or just
the count to trigger more delays.

 # delay the next 5 request by 5 seconds each
 echo 5 5000 > delay

 # set the delay time to 3 seconds without starting a count
 echo 0 3000 > delay

 # delay to the next 5 requests by the current delay time
 echo 5 > delay

Signed-off-by: Chris Leech <cleech@redhat.com>
---
 drivers/nvme/target/Kconfig   |  9 ++++++++
 drivers/nvme/target/debugfs.c | 40 +++++++++++++++++++++++++++++++++++
 drivers/nvme/target/nvmet.h   |  4 ++++
 3 files changed, 53 insertions(+)

diff --git a/drivers/nvme/target/Kconfig b/drivers/nvme/target/Kconfig
index fb7446d6d6829b..e0a1ccb93da68d 100644
--- a/drivers/nvme/target/Kconfig
+++ b/drivers/nvme/target/Kconfig
@@ -126,3 +126,12 @@ config NVME_TARGET_PCI_EPF
 	  capable PCI controller.
 
 	  If unsure, say N.
+
+config NVME_TARGET_DELAY_REQUESTS
+	bool "NVMe over Fabrics target request delay"
+	depends on NVME_TARGET && NVME_TARGET_DEBUGFS
+	help
+	  This is a testing feature to allow delaying request completion in an
+	  NVMe over Fabrics target, which allows for support of the cancel command.
+
+	  If unsure, say N.
diff --git a/drivers/nvme/target/debugfs.c b/drivers/nvme/target/debugfs.c
index e4300eb95101a7..4a80cfceab3460 100644
--- a/drivers/nvme/target/debugfs.c
+++ b/drivers/nvme/target/debugfs.c
@@ -153,6 +153,42 @@ static int nvmet_ctrl_tls_concat_show(struct seq_file *m, void *p)
 NVMET_DEBUGFS_ATTR(nvmet_ctrl_tls_concat);
 #endif
 
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+static int nvmet_ctrl_delay_show(struct seq_file *m, void *p)
+{
+	struct nvmet_ctrl *ctrl = m->private;
+	int delay_count = atomic_read(&ctrl->delay_count);
+
+	seq_printf(m, "%u %u\n", delay_count, ctrl->delay_msec);
+	return 0;
+}
+
+static ssize_t nvmet_ctrl_delay_write(struct file *file, const char __user *buf,
+				      size_t count, loff_t *ppos)
+{
+	struct seq_file *m = file->private_data;
+	struct nvmet_ctrl *ctrl = m->private;
+	char delay_buf[22] = {};
+	int delay_count;
+	int delay_msec;
+	int n;
+
+	if (count >= sizeof(delay_buf))
+		return -EINVAL;
+	if (copy_from_user(delay_buf, buf, count))
+		return -EFAULT;
+
+	n = sscanf(delay_buf, "%u %u", &delay_count, &delay_msec);
+	if (n < 1 || n > 2)
+		return -EINVAL;
+	if (n == 2)
+		ctrl->delay_msec = delay_msec;
+	atomic_set(&ctrl->delay_count, delay_count);
+	return count;
+}
+NVMET_DEBUGFS_RW_ATTR(nvmet_ctrl_delay);
+#endif /* CONFIG_NVME_TARGET_DELAY_REQUESTS */
+
 int nvmet_debugfs_ctrl_setup(struct nvmet_ctrl *ctrl)
 {
 	char name[32];
@@ -183,6 +219,10 @@ int nvmet_debugfs_ctrl_setup(struct nvmet_ctrl *ctrl)
 			    &nvmet_ctrl_tls_concat_fops);
 	debugfs_create_file("tls_key", S_IRUSR, ctrl->debugfs_dir, ctrl,
 			    &nvmet_ctrl_tls_key_fops);
+#endif
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+	debugfs_create_file("delay", S_IWUSR, ctrl->debugfs_dir, ctrl,
+			    &nvmet_ctrl_delay_fops);
 #endif
 	return 0;
 }
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index 7599dbb4347cb2..7a360285b25458 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -306,6 +306,10 @@ struct nvmet_ctrl {
 #endif
 #ifdef CONFIG_NVME_TARGET_TCP_TLS
 	struct key		*tls_key;
+#endif
+#ifdef CONFIG_NVME_TARGET_DELAY_REQUESTS
+	atomic_t		delay_count;
+	u32			delay_msec;
 #endif
 	struct nvmet_pr_log_mgr pr_log_mgr;
 };

From 93a1fcfe991e7d38876a7ae14fdc3ef8e50d1952 Mon Sep 17 00:00:00 2001
From: Chris Leech <cleech@redhat.com>
Date: Wed, 15 Jan 2025 20:58:07 -0800
Subject: [PATCH 09/12] nvmet: delay requests

Allow requests to be delayed for testing.

Signed-off-by: Chris Leech <cleech@redhat.com>
---
 drivers/nvme/target/core.c  | 30 ++++++++++++++++++++++++++++++
 drivers/nvme/target/nvmet.h |  8 ++++++++
 2 files changed, 38 insertions(+)

diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index 3e22dacddd76a7..5622b0608d55b0 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -6,7 +6,9 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 #include <linux/module.h>
 #include <linux/random.h>
+#include <linux/jiffies.h>
 #include <linux/rculist.h>
+#include <linux/workqueue.h>
 #include <linux/pci-p2pdma.h>
 #include <linux/scatterlist.h>
 
@@ -1753,6 +1755,34 @@ ssize_t nvmet_ctrl_host_traddr(struct nvmet_ctrl *ctrl,
 	return ctrl->ops->host_traddr(ctrl, traddr, traddr_len);
 }
 
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+static void nvmet_delayed_execute_req(struct work_struct *work) {
+	struct nvmet_req *req =
+		container_of(to_delayed_work(work), struct nvmet_req, req_work);
+	req->execute(req);
+}
+
+void nvmet_execute_request(struct nvmet_req *req) {
+	struct nvmet_ctrl *ctrl = req->sq->ctrl;
+	int delay_count;
+	u32 delay_msec;
+
+	if (unlikely(req->sq->qid == 0))
+		return req->execute(req);
+
+	if (ctrl) {
+		delay_count = atomic_dec_if_positive(&ctrl->delay_count) + 1;
+		delay_msec = ctrl->delay_msec;
+	}
+	if (!(ctrl && delay_count && delay_msec))
+		return req->execute(req);
+
+	INIT_DELAYED_WORK(&req->req_work, nvmet_delayed_execute_req);
+	queue_delayed_work(nvmet_wq, &req->req_work, msecs_to_jiffies(delay_msec));
+}
+EXPORT_SYMBOL_GPL(nvmet_execute_request);
+#endif
+
 static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port,
 		const char *subsysnqn)
 {
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index 7a360285b25458..1bfdddceda15ca 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -496,6 +496,9 @@ struct nvmet_req {
 	u16			error_loc;
 	u64			error_slba;
 	struct nvmet_pr_per_ctrl_ref *pc_ref;
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+	struct delayed_work	req_work;
+#endif
 };
 
 #define NVMET_MAX_MPOOL_BVEC		16
@@ -971,5 +974,10 @@ struct nvmet_feat_arbitration {
 	u8		ab;
 };
 
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+void nvmet_execute_request(struct nvmet_req *req);
+#else
 static inline void nvmet_execute_request(struct nvmet_req *req) { req->execute(req); }
+#endif
+
 #endif /* _NVMET_H */

From 9670a4f5ac5c7bf7f66b5c27a0fda0de2aef06d6 Mon Sep 17 00:00:00 2001
From: Chris Leech <cleech@redhat.com>
Date: Wed, 15 Jan 2025 16:17:44 -0800
Subject: [PATCH 10/12] nvmet: command tracking

Enable tracking of all outstanding requests in an XArray

Signed-off-by: Chris Leech <cleech@redhat.com>
---
 drivers/nvme/target/core.c  | 29 ++++++++++++++++++++++++++++-
 drivers/nvme/target/nvmet.h |  4 ++++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index 5622b0608d55b0..b446db3804f17b 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -809,10 +809,23 @@ static void __nvmet_req_complete(struct nvmet_req *req, u16 status)
 		nvmet_put_namespace(ns);
 }
 
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+static void nvmet_delayed_execute_req(struct work_struct *work);
+#endif
+
 void nvmet_req_complete(struct nvmet_req *req, u16 status)
 {
 	struct nvmet_sq *sq = req->sq;
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+	unsigned long flags;
 
+	/* only need to update the xarray if this was a delayed request */
+	if (req->req_work.work.func == nvmet_delayed_execute_req) {
+		xa_lock_irqsave(&sq->outstanding_requests, flags);
+		__xa_erase(&sq->outstanding_requests, req->cmd->common.command_id);
+		xa_unlock_irqrestore(&sq->outstanding_requests, flags);
+	}
+#endif
 	__nvmet_req_complete(req, status);
 	percpu_ref_put(&sq->ref);
 }
@@ -984,7 +997,9 @@ int nvmet_sq_init(struct nvmet_sq *sq)
 	init_completion(&sq->free_done);
 	init_completion(&sq->confirm_done);
 	nvmet_auth_sq_init(sq);
-
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+	xa_init_flags(&sq->outstanding_requests, XA_FLAGS_LOCK_IRQ);
+#endif
 	return 0;
 }
 EXPORT_SYMBOL_GPL(nvmet_sq_init);
@@ -1766,6 +1781,7 @@ void nvmet_execute_request(struct nvmet_req *req) {
 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
 	int delay_count;
 	u32 delay_msec;
+	unsigned long flags;
 
 	if (unlikely(req->sq->qid == 0))
 		return req->execute(req);
@@ -1777,6 +1793,17 @@ void nvmet_execute_request(struct nvmet_req *req) {
 	if (!(ctrl && delay_count && delay_msec))
 		return req->execute(req);
 
+	xa_lock_irqsave(&req->sq->outstanding_requests, flags);
+	int ret = __xa_insert(&req->sq->outstanding_requests,
+			    req->cmd->common.command_id, req, GFP_KERNEL);
+	xa_unlock_irqrestore(&req->sq->outstanding_requests, flags);
+
+	if (ret) {
+		pr_err("nvmet: failure to delay command %d",
+		       req->cmd->common.command_id);
+		return req->execute(req);
+	}
+
 	INIT_DELAYED_WORK(&req->req_work, nvmet_delayed_execute_req);
 	queue_delayed_work(nvmet_wq, &req->req_work, msecs_to_jiffies(delay_msec));
 }
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index 1bfdddceda15ca..fc61f28c63daef 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -172,6 +172,9 @@ struct nvmet_sq {
 #endif
 	struct completion	free_done;
 	struct completion	confirm_done;
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+	struct xarray		outstanding_requests;
+#endif
 };
 
 struct nvmet_ana_group {
@@ -573,6 +576,7 @@ size_t nvmet_req_transfer_len(struct nvmet_req *req);
 bool nvmet_check_transfer_len(struct nvmet_req *req, size_t len);
 bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len);
 void nvmet_req_complete(struct nvmet_req *req, u16 status);
+void nvmet_req_complete_delayed(struct nvmet_req *req, u16 status);
 int nvmet_req_alloc_sgls(struct nvmet_req *req);
 void nvmet_req_free_sgls(struct nvmet_req *req);
 

From 96f1828aaafb77391255406ed3cd77556b259911 Mon Sep 17 00:00:00 2001
From: Chris Leech <cleech@redhat.com>
Date: Tue, 4 Feb 2025 10:17:31 -0800
Subject: [PATCH 11/12] nvmet: cancel implementation

Requests are canceled if the have been delayed and cancel_delayed_work
is successfull.

replace emulate_cancel_commands modparam with the delay kconfig

Signed-off-by: Chris Leech <cleech@redhat.com>
---
 drivers/nvme/target/Makefile        |  3 ++-
 drivers/nvme/target/admin-cmd.c     |  7 +++----
 drivers/nvme/target/core.c          |  9 +++------
 drivers/nvme/target/io-cmd-cancel.c | 27 ++++++++++++++++++++++++++-
 drivers/nvme/target/nvmet.h         |  2 --
 5 files changed, 34 insertions(+), 14 deletions(-)

diff --git a/drivers/nvme/target/Makefile b/drivers/nvme/target/Makefile
index a533357e9be4a7..5d9848605285bc 100644
--- a/drivers/nvme/target/Makefile
+++ b/drivers/nvme/target/Makefile
@@ -11,7 +11,8 @@ obj-$(CONFIG_NVME_TARGET_TCP)		+= nvmet-tcp.o
 obj-$(CONFIG_NVME_TARGET_PCI_EPF)	+= nvmet-pci-epf.o
 
 nvmet-y		+= core.o configfs.o admin-cmd.o fabrics-cmd.o \
-			discovery.o io-cmd-file.o io-cmd-bdev.o pr.o io-cmd-cancel.o
+			discovery.o io-cmd-file.o io-cmd-bdev.o pr.o
+nvmet-$(CONFIG_NVME_TARGET_DELAY_REQUESTS) += io-cmd-cancel.o
 nvmet-$(CONFIG_NVME_TARGET_DEBUGFS)	+= debugfs.o
 nvmet-$(CONFIG_NVME_TARGET_PASSTHRU)	+= passthru.o
 nvmet-$(CONFIG_BLK_DEV_ZONED)		+= zns.o
diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c
index 57b90263a7cb95..149c72d0b42958 100644
--- a/drivers/nvme/target/admin-cmd.c
+++ b/drivers/nvme/target/admin-cmd.c
@@ -398,10 +398,9 @@ static void nvmet_get_cmd_effects_nvm(struct nvme_effects_log *log)
 	log->iocs[nvme_cmd_resv_release] =
 	log->iocs[nvme_cmd_resv_report] =
 		cpu_to_le32(NVME_CMD_EFFECTS_CSUPP);
-	if (emulate_cancel_support) {
-		log->iocs[nvme_cmd_cancel] =
-			cpu_to_le32(NVME_CMD_EFFECTS_CSUPP);
-	}
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+	log->iocs[nvme_cmd_cancel] = cpu_to_le32(NVME_CMD_EFFECTS_CSUPP);
+#endif
 	log->iocs[nvme_cmd_write] =
 	log->iocs[nvme_cmd_write_zeroes] =
 		cpu_to_le32(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC);
diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index b446db3804f17b..fd242af9fefbe7 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -29,10 +29,6 @@ static DEFINE_IDA(cntlid_ida);
 struct workqueue_struct *nvmet_wq;
 EXPORT_SYMBOL_GPL(nvmet_wq);
 
-bool emulate_cancel_support;
-module_param(emulate_cancel_support, bool, 0644);
-MODULE_PARM_DESC(emulate_cancel_support, "Emulate the cancel command support. Default = false");
-
 /*
  * This read/write semaphore is used to synchronize access to configuration
  * information on a target system that will result in discovery log page
@@ -1078,12 +1074,13 @@ static u16 nvmet_parse_io_cmd(struct nvmet_req *req)
 	if (nvmet_is_passthru_req(req))
 		return nvmet_parse_passthru_io_cmd(req);
 
-	if (emulate_cancel_support &&
-	    req->cmd->common.opcode == nvme_cmd_cancel) {
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+	if (req->cmd->common.opcode == nvme_cmd_cancel) {
 		req->execute = nvmet_execute_cancel;
 		if (req->cmd->cancel.nsid == NVME_NSID_ALL)
 			return 0;
 	}
+#endif
 
 	ret = nvmet_req_find_ns(req);
 	if (unlikely(ret))
diff --git a/drivers/nvme/target/io-cmd-cancel.c b/drivers/nvme/target/io-cmd-cancel.c
index 8f55458e646a25..8d7f363c916376 100644
--- a/drivers/nvme/target/io-cmd-cancel.c
+++ b/drivers/nvme/target/io-cmd-cancel.c
@@ -14,6 +14,8 @@ void nvmet_execute_cancel(struct nvmet_req *req)
 	int ret = 0;
 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
 	struct nvmet_sq *sq = req->sq;
+	struct nvmet_req *treq;
+	u32 canceled = 0;
 
 	if (!nvmet_check_transfer_len(req, 0))
 		return;
@@ -45,8 +47,31 @@ void nvmet_execute_cancel(struct nvmet_req *req)
 		ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
 	}
 
+	if (!mult_cmds) {
+		treq = xa_load(&sq->outstanding_requests, cid);
+		if (treq) {
+			if (cancel_delayed_work(&treq->req_work)) {
+				pr_info("nvmet: CANCEL success: %d", cid);
+				nvmet_req_complete(treq, NVME_SC_ABORT_REQ);
+				canceled += 1;
+			} else {
+				pr_info("nvmet: CANCEL failed: %d", cid);
+			}
+		} else {
+			pr_info("nvmet: CANCEL request not found: %d", cid);
+		}
+	} else {
+		unsigned long ucid;
+		xa_for_each(&sq->outstanding_requests, ucid, treq) {
+			if (cancel_delayed_work(&treq->req_work)) {
+				nvmet_req_complete(treq, NVME_SC_ABORT_REQ);
+				canceled += 1;
+			}
+		}
+		pr_info("nvmet: CANCEL removed %d requests", canceled);
+	}
 exit:
-	nvmet_set_result(req, 0);
+	nvmet_set_result(req, canceled);
 	nvmet_req_complete(req, ret);
 }
 
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index fc61f28c63daef..52580377418392 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -40,8 +40,6 @@
 #define nvmet_for_each_enabled_ns(xa, index, entry) \
 	xa_for_each_marked(xa, index, entry, NVMET_NS_ENABLED)
 
-extern bool emulate_cancel_support;
-
 /*
  * Supported optional AENs:
  */

From 03484e02f88b46ed199fa2632615fa566a484fb7 Mon Sep 17 00:00:00 2001
From: John Meneghini <jmeneghi@redhat.com>
Date: Wed, 26 Mar 2025 13:49:06 -0400
Subject: [PATCH 12/12] nvmet: stop tracking the cancel command

Signed-off-by: John Meneghini <jmeneghi@redhat.com>
---
 drivers/nvme/target/core.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index fd242af9fefbe7..31efd45ac9cb9d 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -1780,7 +1780,8 @@ void nvmet_execute_request(struct nvmet_req *req) {
 	u32 delay_msec;
 	unsigned long flags;
 
-	if (unlikely(req->sq->qid == 0))
+	if (unlikely(req->sq->qid == 0) ||
+			req->cmd->common.opcode == nvme_cmd_cancel)
 		return req->execute(req);
 
 	if (ctrl) {