DAOS-17357 rebuild: use dmg for stop/start, force-stop fix (daos-stack#16890)

Remove use of fault injection for interactive rebuild testing in daos_test -v (rebuild simple tests). Replace with new dmg command helper functions to invoke the control plane commands dmg pool rebuild stop and dmg pool rebuild start. Some common test functions are added and modified to support this. Also, some rebuild engine logic is updated to remove the associated fault injection handling.

Restore the engine rebuild_ults() logic so that it does not loop forever (a change made in initial implementation for fault injection).

Also, update the engine rebuild stop logic so it does not allow "dmg pool rebuild stop --force" to stop a rebuild in op:Fail_reclaim, unless it has been failing repeatedly. For this, new counts are added in struct rebuild_task and struct rebulid_global_pool_tracker to track, for a high level rebuild operation, how many times op:Rebuild, Reclaim, Fail_reclaim, etc. have been run (and how many times op:Rebuild and op:Fail_reclaim have failed).

daos_test -v rebuild_many_objects_with_failure() is updated to specifically use the --force option to exercise arrival of command during Fail_reclaim, and exercise that the rebuild will not be stopped during Fail_reclaim in this instance. i.e., it will work the same as without the --force option, preventing retry of original op:Rebuild after the one/only Fail_reclaim is done.

Signed-off-by: Kenneth Cain <kenneth.cain@hpe.com>

Author: kccain

src/common/tests_dmg_helpers.c

@@ -1206,6 +1206,80 @@ dmg_pool_list(const char *dmg_config_file, const char *group,
 	return rc;
 }
 
+int
+dmg_pool_rebuild_stop(const char *dmg_config_file, const uuid_t uuid, const char *grp, bool force)
+{
+	char                uuid_str[DAOS_UUID_STR_SIZE];
+	int                 argcount = 0;
+	char              **args     = NULL;
+	struct json_object *dmg_out  = NULL;
+	int                 rc       = 0;
+
+	uuid_unparse_lower(uuid, uuid_str);
+	args = cmd_push_arg(args, &argcount, "%s ", uuid_str);
+	if (args == NULL)
+		D_GOTO(out, rc = -DER_NOMEM);
+
+	if (grp != NULL) {
+		args = cmd_push_arg(args, &argcount, "--sys=%s ", grp);
+		if (args == NULL)
+			D_GOTO(out, rc = -DER_NOMEM);
+	}
+
+	if (force) {
+		args = cmd_push_arg(args, &argcount, "--force");
+		if (args == NULL)
+			D_GOTO(out, rc = -DER_NOMEM);
+	}
+
+	rc = daos_dmg_json_pipe("pool rebuild stop", dmg_config_file, args, argcount, &dmg_out);
+	if (rc != 0) {
+		D_ERROR("dmg pool rebuild stop failed\n");
+		goto out_json;
+	}
+
+out_json:
+	if (dmg_out != NULL)
+		json_object_put(dmg_out);
+	cmd_free_args(args, argcount);
+out:
+	return rc;
+}
+
+int
+dmg_pool_rebuild_start(const char *dmg_config_file, const uuid_t uuid, const char *grp)
+{
+	char                uuid_str[DAOS_UUID_STR_SIZE];
+	int                 argcount = 0;
+	char              **args     = NULL;
+	struct json_object *dmg_out  = NULL;
+	int                 rc       = 0;
+
+	uuid_unparse_lower(uuid, uuid_str);
+	args = cmd_push_arg(args, &argcount, "%s ", uuid_str);
+	if (args == NULL)
+		D_GOTO(out, rc = -DER_NOMEM);
+
+	if (grp != NULL) {
+		args = cmd_push_arg(args, &argcount, "--sys=%s ", grp);
+		if (args == NULL)
+			D_GOTO(out, rc = -DER_NOMEM);
+	}
+
+	rc = daos_dmg_json_pipe("pool rebuild start", dmg_config_file, args, argcount, &dmg_out);
+	if (rc != 0) {
+		D_ERROR("dmg pool rebuild start failed\n");
+		goto out_json;
+	}
+
+out_json:
+	if (dmg_out != NULL)
+		json_object_put(dmg_out);
+	cmd_free_args(args, argcount);
+out:
+	return rc;
+}
+
 static int
 parse_device_info(struct json_object *smd_dev, device_list *devices,
 		  char *host, int dev_length, int *disks)

src/include/daos/common.h

@@ -789,9 +789,6 @@ enum {
 #define DAOS_RDB_SKIP_APPENDENTRIES_FAIL (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1a)
 #define DAOS_FORCE_REFRESH_POOL_MAP	  (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1b)
 
-#define DAOS_REBUILD_ADMIN_STOP           (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1c)
-#define DAOS_REBUILD_ADMIN_START          (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1d)
-
 #define DAOS_FORCE_CAPA_FETCH		(DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1e)
 #define DAOS_FORCE_PROP_VERIFY		(DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1f)
 

src/include/daos/tests_lib.h

@@ -484,6 +484,25 @@ dmg_pool_set_prop(const char *dmg_config_file,
 int dmg_pool_get_prop(const char *dmg_config_file, const char *label, const uuid_t uuid,
 		      const char *name, char **value);
 
+/**
+ * Interactively stop a pool's currently-running rebuild.
+ * \param dmg_config_file [IN] DMG config file.
+ * \param uuid            [IN] UUID of the pool.
+ * \param grp             [IN] Process set name of the DAOS servers managing the pool.
+ * \param force           [IN] forcibly stop a rebuild that is failing.
+ */
+int
+dmg_pool_rebuild_stop(const char *dmg_config_file, const uuid_t uuid, const char *grp, bool force);
+
+/**
+ * Interactively start/resume a pool's rebuilding.
+ * \param dmg_config_file [IN] DMG config file.
+ * \param uuid            [IN] UUID of the pool.
+ * \param grp             [IN] Process set name of the DAOS servers managing the pool.
+ */
+int
+    dmg_pool_rebuild_start(const char *dmg_config_file, const uuid_t uuid, const char *grp);
+
 /**
  * List all disks in the specified DAOS system.
  *

src/include/daos_srv/rebuild.h

@@ -87,7 +87,7 @@ int
      ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver, daos_epoch_t stable_eph,
 			 uint32_t layout_version, struct pool_target_id_list *tgts,
 			 daos_rebuild_opc_t rebuild_op, daos_rebuild_opc_t retry_rebuild_op,
-			 uint32_t retry_map_ver, bool stop_admin, uint64_t delay_sec);
+			 uint32_t retry_map_ver, bool stop_admin, void *cur_taskp, uint64_t delay_sec);
 void ds_rebuild_restart_if_rank_wip(uuid_t pool_uuid, d_rank_t rank);
 int ds_rebuild_query(uuid_t pool_uuid,
 		     struct daos_rebuild_status *status);

src/pool/srv_pool.c

@@ -6410,7 +6410,7 @@ pool_check_upgrade_object_layout(struct rdb_tx *tx, struct pool_svc *svc,
 		rc = ds_rebuild_schedule(svc->ps_pool, svc->ps_pool->sp_map_version, upgrade_eph,
 					 DAOS_POOL_OBJ_VERSION, NULL, RB_OP_UPGRADE,
 					 RB_OP_NONE /* retry_rebuild_op */, 0 /* retry_map_ver */,
-					 false /* stop_admin */, 0);
+					 false /* stop_admin */, NULL /* cur_taskp */, 0);
 		if (rc == 0)
 			*scheduled_layout_upgrade = true;
 	}
@@ -7758,7 +7758,8 @@ pool_svc_update_map(struct pool_svc *svc, crt_opcode_t opc, bool exclude_rank,
 	if (tgt_map_ver != 0) {
 		rc = ds_rebuild_schedule(svc->ps_pool, tgt_map_ver, rebuild_eph, 0, &target_list,
 					 RB_OP_REBUILD, RB_OP_NONE /* retry_rebuild_op */,
-					 0 /* retry_map_ver */, false /* stop_admin */, delay);
+					 0 /* retry_map_ver */, false /* stop_admin */,
+					 NULL /* cur_taskp */, delay);
 		if (rc != 0) {
 			D_ERROR("rebuild fails rc: "DF_RC"\n", DP_RC(rc));
 			D_GOTO(out, rc);

src/rebuild/rebuild_internal.h

@@ -168,6 +168,11 @@ struct rebuild_global_pool_tracker {
 	uint32_t	rgt_opc;
 	unsigned int                    rgt_abort : 1, /* abort: kill rebuild */
 	    rgt_init_scan : 1, rgt_stop_admin : 1;     /* stop: admin has asked to kill rebuild */
+
+	uint32_t rgt_num_op_rb;            /* count of op:Rebuild attempts */
+	uint32_t rgt_num_op_freclaim;      /* count of op:Fail_reclaim attempts */
+	uint32_t rgt_num_op_rb_fail;       /* count of op:Rebuild failures */
+	uint32_t rgt_num_op_freclaim_fail; /* count of failed op:Fail_reclaim (not good) */
 };
 
 /* Structure on raft replica nodes to serve completed rebuild status querying */
@@ -254,6 +259,14 @@ struct rebuild_task {
 	 * Then, on fail_reclaim finish, the pool rebuild state will be set to idle (NOT_STARTED).
 	 */
 	bool                            dst_stop_admin;
+
+	/* Track how many tries for certain daos_rebuild_opc_t */
+	uint32_t                        dst_num_op_rb; /* count of tries to run rebuild */
+	uint32_t                        dst_num_op_reclaim;
+	uint32_t                        dst_num_op_freclaim;
+	uint32_t                        dst_num_op_upgrade;
+	uint32_t                        dst_num_op_rb_fail; /* count of rebuild failures */
+	uint32_t dst_num_op_freclaim_fail;                  /* count of Fail_recliam failures */
 };
 
 /* Per pool structure in TLS to check pool rebuild status