DAOS-18976 rebuild: refine rebuild gen handling (#18380)

1. For RECLAIM/FAIL_RECLAIM always bump the rebuild gen.
2. abort local rpt when rgt done.
Some log refinning

Signed-off-by: Xuezhao Liu <xuezhao.liu@hpe.com>
Co-authored-by: Liang Zhen <gnailzenh@gmail.com>

Author: liuxuezhao

src/rebuild/rebuild_iv.c

@@ -1,6 +1,6 @@
 /**
  * (C) Copyright 2017-2024 Intel Corporation.
- * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+ * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -188,8 +188,8 @@ rebuild_iv_ent_refresh(struct ds_iv_entry *entry, struct ds_iv_key *key,
 
 	if (ref_rc != 0) {
 		rc = ref_rc;
-		DL_WARN(rc, DF_UUID "bypass refresh, IV class id %d.",
-			DP_UUID(entry->ns->iv_pool_uuid), key->class_id);
+		DL_WARN(rc, DF_RB ", IV ns pool " DF_UUID "bypass refresh, IV class id %d.",
+			DP_RB_RPT(rpt), DP_UUID(entry->ns->iv_pool_uuid), key->class_id);
 		goto out;
 	}
 

src/rebuild/scan.c

@@ -131,14 +131,17 @@ rebuild_obj_send_cb(struct tree_cache_root *root, struct rebuild_send_arg *arg)
 
 		if (rpt->rt_abort || rpt->rt_finishing || rpt->rt_global_done) {
 			rc = -DER_SHUTDOWN;
-			DL_INFO(rc, DF_RB ": give up ds_object_migrate_send, shutdown rebuild",
-				DP_RB_RPT(rpt));
+			DL_INFO(rc,
+				DF_RB ": rt_abort %d, rt_finishing %d, rt_global_done %d, "
+				      "give up ds_object_migrate_send, shutdown rebuild",
+				DP_RB_RPT(rpt), rpt->rt_abort, rpt->rt_finishing,
+				rpt->rt_global_done);
 			break;
 		}
 
 		/* otherwise let's retry */
-		D_DEBUG(DB_REBUILD, DF_UUID" retry send object to tgt_id %d\n",
-			DP_UUID(rpt->rt_pool_uuid), arg->tgt_id);
+		D_DEBUG(DB_REBUILD, DF_RB " retry send object to tgt_id %d\n", DP_RB_RPT(rpt),
+			arg->tgt_id);
 		dss_sleep(daos_rpc_rand_delay(max_delay) << 10);
 	}
 out:
@@ -360,8 +363,10 @@ rebuild_objects_send_ult(void *data)
 		D_FREE(ephs);
 	if (punched_ephs != NULL)
 		D_FREE(punched_ephs);
-	if (rc != 0 && tls->rebuild_pool_status == 0)
+	if (rc != 0 && tls->rebuild_pool_status == 0) {
+		DL_ERROR(rc, DF_RB " set rebuild_pool_status as failed", DP_RB_RPT(rpt));
 		tls->rebuild_pool_status = rc;
+	}
 
 	rpt_put(rpt);
 }
@@ -1305,6 +1310,7 @@ rebuild_tgt_scan_handler(crt_rpc_t *rpc)
 			rpt->rt_re_report = 1;
 
 			rpt->rt_leader_rank = rsi->rsi_master_rank;
+			rpt->rt_rebuild_gen = rsi->rsi_rebuild_gen;
 
 			/* If this is the old leader, then also stop the rebuild tracking ULT. */
 			rebuild_leader_stop(rsi->rsi_pool_uuid, rsi->rsi_rebuild_ver,

src/rebuild/srv.c

@@ -1459,7 +1459,8 @@ rebuild_leader_start(struct ds_pool *pool, struct rebuild_task *task,
 	 */
 	ds_rebuild_running_query_adv(pool->sp_uuid, -1, &version, NULL, &generation,
 				     &rebuild_leader_rank, &rebuild_leader_term);
-	if ((version < task->dst_map_ver) ||
+	if (task->dst_rebuild_op == RB_OP_RECLAIM || task->dst_rebuild_op == RB_OP_FAIL_RECLAIM ||
+	    (version < task->dst_map_ver) ||
 	    (version == task->dst_map_ver && leader_rank == rebuild_leader_rank &&
 	     leader_term == rebuild_leader_term))
 		generation = ++pool->sp_rebuild_gen;
@@ -1806,6 +1807,8 @@ rebuild_task_ult(void *arg)
 	 * rebuild.
 	 */
 	if (rgt && rgt->rgt_init_scan) {
+		struct rebuild_tgt_pool_tracker *local_rpt;
+
 		if (myrank != pool->sp_iv_ns->iv_master_rank) {
 			/* If master has been changed, then let's skip
 			 * iv sync, and the new leader will take over
@@ -1817,6 +1820,14 @@ rebuild_task_ult(void *arg)
 		}
 
 		rebuild_leader_status_notify(rgt, pool, task->dst_rebuild_op, myrank);
+
+		local_rpt = rpt_lookup(pool->sp_uuid, task->dst_rebuild_op, rgt->rgt_rebuild_ver,
+				       rgt->rgt_rebuild_gen);
+		if (local_rpt) {
+			local_rpt->rt_abort = 1;
+			D_INFO(DF_RB " set rt_abort", DP_RB_RPT(local_rpt));
+			rpt_put(local_rpt);
+		}
 	}
 
 try_reschedule:
@@ -2385,8 +2396,7 @@ rebuild_tgt_fini(struct rebuild_tgt_pool_tracker *rpt)
 	/* destroy the migrate_tls of 0-xstream */
 	ds_migrate_stop(rpt->rt_pool, rpt->rt_rebuild_ver, rpt->rt_rebuild_gen);
 	/* No one should access rpt after rebuild_fini_one. */
-	D_INFO("Finalized rebuild for "DF_UUID", map_ver=%u.\n",
-	       DP_UUID(rpt->rt_pool_uuid), rpt->rt_rebuild_ver);
+	DL_INFO(rc, DF_RB " Finalized rebuild", DP_RB_RPT(rpt));
 	rpt_delete(rpt);
 }
 
@@ -2515,8 +2525,12 @@ rebuild_tgt_status_check_ult(void *arg)
 				 * it can not find the IV see crt_iv_hdlr_xx().
 				 * let's just stop the rebuild.
 				 */
-				if (rc == -DER_NONEXIST && !status.rebuilding)
+				if (rc == -DER_NONEXIST && !status.rebuilding) {
+					D_INFO(DF_RB ", rc %d, status.rebuilding %d, "
+						     "set rt_global_done",
+					       DP_RB_RPT(rpt), rc, status.rebuilding);
 					rpt->rt_global_done = 1;
+				}
 
 				if (ns->iv_stop) {
 					D_DEBUG(DB_REBUILD, "abort rebuild "