lib/nvmf: Add vendor specific subsystem pause/resume

doc/jsonrpc.md

@@ -8792,6 +8792,120 @@ Example response:
 }
 ~~~
 
+### nvmf_subsystem_set_pause_timeout method {#rpc_nvmf_subsystem_set_pause_timeout}
+
+Set the pause timeout value for a subsystem.
+
+#### Parameters
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+nqn                     | Required | string      | Subsystem NQN
+pause_timeout_sec       | Required | number      | Pause timeout in seconds
+tgt_name                | Optional | string      | Parent NVMe-oF target name.
+
+#### Example
+
+Example request:
+
+~~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "nvmf_subsystem_set_pause_timeout",
+  "params": {
+    "nqn": "nqn.2016-06.io.spdk:cnode1",
+    "pause_timeout_sec": 120
+  }
+}
+~~
+
+Example response:
+
+~~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": true
+}
+~~
+
+### nvmf_subsystem_pause_ext method {#rpc_nvmf_subsystem_pause_ext}
+
+Pause an NVMe-oF subsystem with the specified pause flags.
+
+#### Parameters
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+nqn                     | Required | string      | Subsystem NQN
+nsid                    | Optional | number      | Namespace ID. Defaults to 0.
+flags                   | Optional | number      | Pause operation flags. Defaults to 0.
+tgt_name                | Optional | string      | Parent NVMe-oF target name.
+
+#### Example
+
+Example request:
+
+~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "nvmf_subsystem_pause_ext",
+  "params": {
+    "nqn": "nqn.2016-06.io.spdk:cnode1",
+    "nsid": 1,
+    "flags": 0
+  }
+}
+~
+
+Example response:
+
+~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": true
+}
+~
+
+### nvmf_subsystem_resume_ext method {#rpc_nvmf_subsystem_resume_ext}
+
+Resume an NVMe-oF subsystem previously paused with `nvmf_subsystem_pause_ext`.
+
+#### Parameters
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+nqn                     | Required | string      | Subsystem NQN
+tgt_name                | Optional | string      | Parent NVMe-oF target name.
+
+#### Example
+
+Example request:
+
+~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "nvmf_subsystem_resume_ext",
+  "params": {
+    "nqn": "nqn.2016-06.io.spdk:cnode1"
+  }
+}
+~
+
+Example response:
+
+~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": true
+}
+~
+
 ### nvmf_subsystem_add_host method {#rpc_nvmf_subsystem_add_host}
 
 Add a host NQN to the list of allowed hosts.  Adding an already allowed host will result in an

doc/nvmf_tgt_pg.md

@@ -75,7 +75,8 @@ Subsystems begin in an inactive state and must be activated by calling
 spdk_nvmf_subsystem_start(). Subsystems may be modified at run time, but only
 when in the paused or inactive state. A running subsystem may be paused by
 calling spdk_nvmf_subsystem_pause() and resumed by calling
-spdk_nvmf_subsystem_resume().
+spdk_nvmf_subsystem_resume(). Also nvmf_subsystem_pause_ext() and
+nvmf_subsystem_resume_ext() is vendor specific method for pause and resume.
 
 Namespaces may be added to the subsystem by calling
 spdk_nvmf_subsystem_add_ns_ext() when the subsystem is inactive or paused.

go/rpc/clientIntegration.go

@@ -46,7 +46,9 @@ func spdk_gorpc_call(jsonPtr *C.char, location *C.char) (*C.char, C.int) {
 		log.Printf("error on client creation, err: %s", err.Error())
 		return nil, ConnectionError
 	}
-	defer rpcClient.Close()
+	defer func() {
+		_ = rpcClient.Close()
+	}()
 
 	method := jsonMap["method"].(string)
 	params := jsonMap["params"].(map[string]any)

include/spdk/nvmf.h

@@ -27,6 +27,8 @@ extern "C" {
 
 #define SPDK_TLS_PSK_MAX_LEN		200
 
+#define SPDK_NVMF_SUBSYSTEM_PAUSE_KEEP_ADMINQ 0x1
+
 struct spdk_nvmf_tgt;
 struct spdk_nvmf_subsystem;
 struct spdk_nvmf_ctrlr;
@@ -566,6 +568,21 @@ int spdk_nvmf_subsystem_pause(struct spdk_nvmf_subsystem *subsystem,
 			      spdk_nvmf_subsystem_state_change_done cb_fn,
 			      void *cb_arg);
 
+/**
+ * Vendor specific wrapper of subsystem pause
+ * we need to keep allow some admin commands during subsystem pause
+ */
+int spdk_nvmf_subsystem_pause_ext(struct spdk_nvmf_subsystem *subsystem,
+				  uint32_t nsid,
+				  uint32_t flags,
+				  spdk_nvmf_subsystem_state_change_done cb_fn,
+				  void *cb_arg);
+
+int spdk_nvmf_subsystem_set_pause_timeout(struct spdk_nvmf_subsystem *subsystem,
+		uint32_t timeout_sec);
+
+uint32_t spdk_nvmf_subsystem_get_pause_timeout(struct spdk_nvmf_subsystem *subsystem);
+
 /**
  * Transition an NVMe-oF subsystem from Paused to Active state.
  *
@@ -582,6 +599,13 @@ int spdk_nvmf_subsystem_resume(struct spdk_nvmf_subsystem *subsystem,
 			       spdk_nvmf_subsystem_state_change_done cb_fn,
 			       void *cb_arg);
 
+/**
+ * Vendor specific wrapper of subsystem resume
+ * We need to clear vendor specific flags during resume
+ */
+int spdk_nvmf_subsystem_resume_ext(struct spdk_nvmf_subsystem *subsystem,
+				   spdk_nvmf_subsystem_state_change_done cb_fn,
+				   void *cb_arg);
 /**
  * Search the target for a subsystem with the given NQN.
  *

lib/nvmf/ctrlr.c

@@ -4859,6 +4859,7 @@ nvmf_check_subsystem_active(struct spdk_nvmf_request *req)
 	struct spdk_nvmf_subsystem_poll_group *sgroup = NULL;
 	struct spdk_nvmf_subsystem_pg_ns_info *ns_info;
 	uint32_t nsid;
+	struct spdk_nvmf_subsystem *subsystem;
 
 	if (spdk_likely(qpair->ctrlr)) {
 		sgroup = &qpair->group->sgroups[qpair->ctrlr->subsys->id];
@@ -4875,6 +4876,16 @@ nvmf_check_subsystem_active(struct spdk_nvmf_request *req)
 	if (spdk_unlikely(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC ||
 			  nvmf_qpair_is_admin_queue(qpair))) {
 		if (sgroup->state != SPDK_NVMF_SUBSYSTEM_ACTIVE) {
+			subsystem = qpair->ctrlr->subsys;
+			if (req->cmd->nvmf_cmd.opcode != SPDK_NVME_OPC_FABRIC &&
+			    (subsystem->pause_flags & SPDK_NVMF_SUBSYSTEM_PAUSE_KEEP_ADMINQ)) {
+				/*
+				 * Vendor maintenance pause
+				 * allow normal admin queue commands while data IO is paused
+				 */
+				sgroup->mgmt_io_outstanding++;
+				return true;
+			}
 			/* The subsystem is not currently active. Queue this request. */
 			TAILQ_INSERT_TAIL(&sgroup->queued, req, link);
 			return false;

lib/nvmf/nvmf_internal.h

@@ -26,7 +26,7 @@
 /* The spec reserves cntlid values in the range FFF0h to FFFFh. */
 #define NVMF_MIN_CNTLID 1
 #define NVMF_MAX_CNTLID 0xFFEF
-
+#define SPDK_NVMF_DEFAULT_PAUSE_TIMEOUT_SEC 120
 #define NVMF_DISC_KATO_IN_MS 120000
 #define NVMF_KAS_TIME_UNIT_IN_MS 100
 #define NVMF_DEFAULT_KAS 100
@@ -365,6 +365,10 @@ struct spdk_nvmf_subsystem {
 	/* Subsystem event callback and its argument. */
 	spdk_nvmf_subsystem_event_cb			event_cb_fn;
 	void						*event_cb_arg;
+	/* Vendor specific flags for maintainance work */
+	uint32_t                                        pause_flags;
+	struct spdk_poller				*pause_timer;
+	uint32_t					pause_timeout_sec;
 };
 
 static int

lib/nvmf/subsystem.c

@@ -29,6 +29,11 @@
 #define MODEL_NUMBER_DEFAULT "SPDK bdev Controller"
 #define NVMF_SUBSYSTEM_DEFAULT_NAMESPACES 32
 
+struct spdk_nvmf_resume_ext_ctx {
+	spdk_nvmf_subsystem_state_change_done cb_fn;
+	void *cb_arg;
+};
+
 /*
  * States for parsing valid domains in NQNs according to RFC 1034
  */
@@ -265,6 +270,9 @@ spdk_nvmf_subsystem_create(struct spdk_nvmf_tgt *tgt,
 	subsystem->next_cntlid = 1;
 	subsystem->min_cntlid = NVMF_MIN_CNTLID;
 	subsystem->max_cntlid = NVMF_MAX_CNTLID;
+	subsystem->pause_timeout_sec = SPDK_NVMF_DEFAULT_PAUSE_TIMEOUT_SEC;
+	subsystem->pause_flags = 0;
+	subsystem->pause_timer = NULL;
 	snprintf(subsystem->subnqn, sizeof(subsystem->subnqn), "%s", nqn);
 	pthread_mutex_init(&subsystem->mutex, NULL);
 	TAILQ_INIT(&subsystem->listeners);
@@ -400,6 +408,11 @@ _nvmf_subsystem_destroy(struct spdk_nvmf_subsystem *subsystem)
 		return -EINPROGRESS;
 	}
 
+	if (subsystem->pause_timer) {
+		spdk_poller_unregister(&subsystem->pause_timer);
+		subsystem->pause_timer = NULL;
+	}
+
 	ns = spdk_nvmf_subsystem_get_first_ns(subsystem);
 	while (ns != NULL) {
 		struct spdk_nvmf_ns *next_ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns);
@@ -813,6 +826,51 @@ spdk_nvmf_subsystem_stop(struct spdk_nvmf_subsystem *subsystem,
 	return nvmf_subsystem_state_change(subsystem, 0, SPDK_NVMF_SUBSYSTEM_INACTIVE, cb_fn, cb_arg);
 }
 
+static
+int
+nvmf_subsys_pause_timer_cb(void *arg)
+{
+	struct spdk_nvmf_subsystem *subsystem = arg;
+	subsystem->pause_flags = 0;
+	if (subsystem->pause_timer) {
+		spdk_poller_unregister(&subsystem->pause_timer);
+		subsystem->pause_timer = NULL;
+	}
+	return SPDK_POLLER_BUSY;
+}
+
+int
+spdk_nvmf_subsystem_pause_ext(struct spdk_nvmf_subsystem *subsystem,
+			      uint32_t nsid,
+			      uint32_t flags,
+			      spdk_nvmf_subsystem_state_change_done cb_fn,
+			      void *cb_arg)
+{
+	subsystem->pause_flags = flags;
+	if (subsystem->pause_timer == NULL) {
+		subsystem->pause_timer = spdk_poller_register(
+						 nvmf_subsys_pause_timer_cb,
+						 subsystem,
+						 subsystem->pause_timeout_sec * 1000000ULL
+					 );
+	}
+	return spdk_nvmf_subsystem_pause(subsystem, nsid, cb_fn, cb_arg);
+}
+
+int
+spdk_nvmf_subsystem_set_pause_timeout(struct spdk_nvmf_subsystem *subsystem,
+				      uint32_t timeout_sec)
+{
+	subsystem->pause_timeout_sec = timeout_sec;
+	return 0;
+}
+
+uint32_t
+spdk_nvmf_subsystem_get_pause_timeout(struct spdk_nvmf_subsystem *subsystem)
+{
+	return subsystem->pause_timeout_sec;
+}
+
 int
 spdk_nvmf_subsystem_pause(struct spdk_nvmf_subsystem *subsystem,
 			  uint32_t nsid,
@@ -822,11 +880,54 @@ spdk_nvmf_subsystem_pause(struct spdk_nvmf_subsystem *subsystem,
 	return nvmf_subsystem_state_change(subsystem, nsid, SPDK_NVMF_SUBSYSTEM_PAUSED, cb_fn, cb_arg);
 }
 
+static void
+nvmf_subsystem_resume_ext_done(struct spdk_nvmf_subsystem *subsystem,
+			       void *cb_arg, int status)
+{
+	struct spdk_nvmf_resume_ext_ctx *ctx = cb_arg;
+
+	if (status == 0) {
+		subsystem->pause_flags = 0;
+
+		if (subsystem->pause_timer != NULL) {
+			spdk_poller_unregister(&subsystem->pause_timer);
+		}
+	}
+
+	ctx->cb_fn(subsystem, ctx->cb_arg, status);
+	free(ctx);
+}
+
+int
+spdk_nvmf_subsystem_resume_ext(struct spdk_nvmf_subsystem *subsystem,
+			       spdk_nvmf_subsystem_state_change_done cb_fn,
+			       void *cb_arg)
+{
+	struct spdk_nvmf_resume_ext_ctx *ctx;
+
+	ctx = calloc(1, sizeof(*ctx));
+	if (ctx == NULL) {
+		return -ENOMEM;
+	}
+
+	ctx->cb_fn = cb_fn;
+	ctx->cb_arg = cb_arg;
+
+	return spdk_nvmf_subsystem_resume(subsystem,
+					  nvmf_subsystem_resume_ext_done,
+					  ctx);
+}
+
 int
 spdk_nvmf_subsystem_resume(struct spdk_nvmf_subsystem *subsystem,
 			   spdk_nvmf_subsystem_state_change_done cb_fn,
 			   void *cb_arg)
 {
+	subsystem->pause_flags = 0;
+	if (subsystem->pause_timer) {
+		spdk_poller_unregister(&subsystem->pause_timer);
+		subsystem->pause_timer = NULL;
+	}
 	return nvmf_subsystem_state_change(subsystem, 0, SPDK_NVMF_SUBSYSTEM_ACTIVE, cb_fn, cb_arg);
 }