Merge remote-tracking branch 'origin/release/2.6' into grom72/SRE-3703-2.6

Signed-off-by: Tomasz Gromadzki <tomasz.gromadzki@hpe.com>

Author: grom72

src/object/srv_obj_migrate.c

@@ -271,6 +271,17 @@ struct iter_obj_arg {
 	uint32_t		generation;
 };
 
+#define MIGR_RETRY_WAIT_WARN(tls, oid, rc, tried, duration)                                        \
+	do {                                                                                       \
+		tried++;                                                                           \
+		if (tried >= 4096)                                                                 \
+			tried = 2048;                                                              \
+		if ((tried & (tried - 1)) == 0)                                                    \
+			DL_WARN(rc, DF_RB ": retry " DF_UOID ", tried[%d] " DF_U64 " seconds ",    \
+				DP_RB_MPT(tls), DP_UOID(oid), tried, (duration));                  \
+		dss_sleep(1000);                                                                   \
+	} while (0)
+
 static int
 migrate_try_obj_insert(struct migrate_pool_tls *tls, uuid_t co_uuid, daos_unit_oid_t oid,
 		       daos_epoch_t epoch, daos_epoch_t punched_epoch, unsigned int shard,
@@ -778,6 +789,7 @@ mrone_obj_fetch_internal(struct migrate_one *mrone, daos_handle_t oh, d_sg_list_
 	uint64_t              now;
 	int       rc;
 	int                   wait = MEM_NO_WAIT;
+	int                   tried = 0;
 
 	/* pass rebuild epoch by extra_arg */
 	if (flags & DIOF_FETCH_EPOCH_EC_AGG_BOUNDARY) {
@@ -789,7 +801,8 @@ mrone_obj_fetch_internal(struct migrate_one *mrone, daos_handle_t oh, d_sg_list_
 retry:
 	rc = dsc_obj_fetch(oh, eph, &mrone->mo_dkey, iod_num, iods, sgls, NULL, flags, extra_arg,
 			   csum_iov_fetch);
-	if ((rc == -DER_TIMEDOUT || rc == -DER_FETCH_AGAIN || rc == -DER_NOMEM) &&
+	if ((rc == -DER_TIMEDOUT || rc == -DER_FETCH_AGAIN || rc == -DER_NOMEM ||
+	     daos_crt_network_error(rc)) &&
 	    tls->mpt_version + 1 >= tls->mpt_pool->spc_map_version) {
 		if (tls->mpt_fini) {
 			DL_ERROR(rc, DF_RB ": dsc_obj_fetch " DF_UOID "failed when mpt_fini",
@@ -799,19 +812,25 @@ mrone_obj_fetch_internal(struct migrate_one *mrone, daos_handle_t oh, d_sg_list_
 		/* If pool map does not change, then let's retry for timeout, instead of
 		 * fail out.
 		 */
+		now = daos_gettime_coarse();
+		if (then == 0)
+			then = now;
+
 		if (rc != -DER_NOMEM) {
-			DL_WARN(rc, DF_RB ": retry " DF_UOID, DP_RB_MPT(tls),
-				DP_UOID(mrone->mo_oid));
-			dss_sleep(1000);
-			D_GOTO(retry, rc);
+			if (rc == -DER_TIMEDOUT || rc == -DER_FETCH_AGAIN || now - then < 600) {
+				MIGR_RETRY_WAIT_WARN(tls, mrone->mo_oid, rc, tried, now - then);
+				D_GOTO(retry, rc);
+			}
+			/* waited for too long, return error and restart rebuild */
+			DL_ERROR(rc, DF_RB " waited for over 10 minutes due to network error",
+				 DP_RB_MRO(mrone));
+			return rc;
 		}
 
-		now = daos_gettime_coarse();
 		if (wait == MEM_NO_WAIT) {
 			wait = MEM_WAIT;
 			res->res_data.mem_waiting++;
 			res->res_data.mem_err++;
-			then = now;
 		}
 		/* sleep a few seconds before retry, give other layers a chance to
 		 * release resources.
@@ -3081,7 +3100,9 @@ migrate_obj_epoch(struct migrate_pool_tls *tls, struct iter_obj_arg *arg, daos_e
 	uint32_t		 minimum_nr;
 	uint32_t		 enum_flags;
 	uint32_t		 num;
-	int                       waited = 0;
+	uint64_t                  now;
+	uint64_t                  then  = 0;
+	int                       tried = 0;
 	int			 rc = 0;
 
 	D_DEBUG(DB_REBUILD, "migrate obj "DF_UOID" for shard %u eph "
@@ -3216,23 +3237,29 @@ migrate_obj_epoch(struct migrate_pool_tls *tls, struct iter_obj_arg *arg, daos_e
 			/* -DER_UPDATE_AGAIN means the remote target does not parse EC
 			 * aggregation yet, so let's retry.
 			 */
-			waited++;
+			now = daos_gettime_coarse();
+			if (then == 0)
+				then = now;
 			dss_sleep(5000);
 			D_DEBUG(DB_REBUILD, DF_UOID "retry %d secs with %d \n", DP_UOID(arg->oid),
-				waited * 5, rc);
+				(int)(now + 5 - then), rc);
 			rc = 0;
 			continue;
 		} else if (rc) {
 			/* To avoid reclaim and retry rebuild, let's retry until the pool map
 			 * being changed due to further failure.
 			 */
-			if (rc == -DER_TIMEDOUT &&
+			if ((rc == -DER_TIMEDOUT || daos_crt_network_error(rc)) &&
 			    tls->mpt_version + 1 >= tls->mpt_pool->spc_map_version) {
-				D_WARN(DF_UUID" retry "DF_UOID" "DF_RC"\n",
-				       DP_UUID(tls->mpt_pool_uuid), DP_UOID(arg->oid),
-				       DP_RC(rc));
-				rc = 0;
-				continue;
+				now = daos_gettime_coarse();
+				if (then == 0)
+					then = now;
+				if (rc == -DER_TIMEDOUT || now - then < 600) {
+					MIGR_RETRY_WAIT_WARN(tls, arg->oid, rc, tried, now - then);
+					rc = 0;
+					continue;
+				}
+				/* fall through and fail rebuild */
 			}
 
 			/* container might have been destroyed. Or there is

src/rebuild/rebuild_iv.c

@@ -1,6 +1,6 @@
 /**
  * (C) Copyright 2017-2024 Intel Corporation.
- * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+ * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -188,8 +188,8 @@ rebuild_iv_ent_refresh(struct ds_iv_entry *entry, struct ds_iv_key *key,
 
 	if (ref_rc != 0) {
 		rc = ref_rc;
-		DL_WARN(rc, DF_UUID "bypass refresh, IV class id %d.",
-			DP_UUID(entry->ns->iv_pool_uuid), key->class_id);
+		DL_WARN(rc, DF_RB ", IV ns pool " DF_UUID "bypass refresh, IV class id %d.",
+			DP_RB_RPT(rpt), DP_UUID(entry->ns->iv_pool_uuid), key->class_id);
 		goto out;
 	}
 
@@ -214,11 +214,16 @@ rebuild_iv_ent_refresh(struct ds_iv_entry *entry, struct ds_iv_key *key,
 			D_WARN("leader change stable epoch from "DF_U64" to "
 			       DF_U64 "\n", rpt->rt_stable_epoch,
 			       dst_iv->riv_stable_epoch);
-		rpt->rt_global_done = dst_iv->riv_global_done;
-		rpt->rt_global_scan_done = dst_iv->riv_global_scan_done;
+
+		/* NB: IV refresh can arrive out of order, but rebuild can't revert global done */
+		if (!rpt->rt_global_done)
+			rpt->rt_global_done = dst_iv->riv_global_done;
+		if (!rpt->rt_global_scan_done)
+			rpt->rt_global_scan_done = dst_iv->riv_global_scan_done;
 		old_ver = rpt->rt_global_dtx_resync_version;
-		if (rpt->rt_global_dtx_resync_version < dst_iv->riv_global_dtx_resyc_version)
+		if (old_ver < dst_iv->riv_global_dtx_resyc_version)
 			rpt->rt_global_dtx_resync_version = dst_iv->riv_global_dtx_resyc_version;
+
 		if (old_ver < rpt->rt_rebuild_ver &&
 		    dst_iv->riv_global_dtx_resyc_version >= rpt->rt_rebuild_ver) {
 			D_INFO(DF_UUID " global/iv/rebuild_ver %u/%u/%u signal wait cond\n",

src/rebuild/scan.c

@@ -131,14 +131,17 @@ rebuild_obj_send_cb(struct tree_cache_root *root, struct rebuild_send_arg *arg)
 
 		if (rpt->rt_abort || rpt->rt_finishing || rpt->rt_global_done) {
 			rc = -DER_SHUTDOWN;
-			DL_INFO(rc, DF_RB ": give up ds_object_migrate_send, shutdown rebuild",
-				DP_RB_RPT(rpt));
+			DL_INFO(rc,
+				DF_RB ": rt_abort %d, rt_finishing %d, rt_global_done %d, "
+				      "give up ds_object_migrate_send, shutdown rebuild",
+				DP_RB_RPT(rpt), rpt->rt_abort, rpt->rt_finishing,
+				rpt->rt_global_done);
 			break;
 		}
 
 		/* otherwise let's retry */
-		D_DEBUG(DB_REBUILD, DF_UUID" retry send object to tgt_id %d\n",
-			DP_UUID(rpt->rt_pool_uuid), arg->tgt_id);
+		D_DEBUG(DB_REBUILD, DF_RB " retry send object to tgt_id %d\n", DP_RB_RPT(rpt),
+			arg->tgt_id);
 		dss_sleep(daos_rpc_rand_delay(max_delay) << 10);
 	}
 out:
@@ -360,8 +363,10 @@ rebuild_objects_send_ult(void *data)
 		D_FREE(ephs);
 	if (punched_ephs != NULL)
 		D_FREE(punched_ephs);
-	if (rc != 0 && tls->rebuild_pool_status == 0)
+	if (rc != 0 && tls->rebuild_pool_status == 0) {
+		DL_ERROR(rc, DF_RB " set rebuild_pool_status as failed", DP_RB_RPT(rpt));
 		tls->rebuild_pool_status = rc;
+	}
 
 	rpt_put(rpt);
 }
@@ -1305,6 +1310,7 @@ rebuild_tgt_scan_handler(crt_rpc_t *rpc)
 			rpt->rt_re_report = 1;
 
 			rpt->rt_leader_rank = rsi->rsi_master_rank;
+			rpt->rt_rebuild_gen = rsi->rsi_rebuild_gen;
 
 			/* If this is the old leader, then also stop the rebuild tracking ULT. */
 			rebuild_leader_stop(rsi->rsi_pool_uuid, rsi->rsi_rebuild_ver,

src/rebuild/srv.c

@@ -1459,7 +1459,8 @@ rebuild_leader_start(struct ds_pool *pool, struct rebuild_task *task,
 	 */
 	ds_rebuild_running_query_adv(pool->sp_uuid, -1, &version, NULL, &generation,
 				     &rebuild_leader_rank, &rebuild_leader_term);
-	if ((version < task->dst_map_ver) ||
+	if (task->dst_rebuild_op == RB_OP_RECLAIM || task->dst_rebuild_op == RB_OP_FAIL_RECLAIM ||
+	    (version < task->dst_map_ver) ||
 	    (version == task->dst_map_ver && leader_rank == rebuild_leader_rank &&
 	     leader_term == rebuild_leader_term))
 		generation = ++pool->sp_rebuild_gen;
@@ -1806,6 +1807,8 @@ rebuild_task_ult(void *arg)
 	 * rebuild.
 	 */
 	if (rgt && rgt->rgt_init_scan) {
+		struct rebuild_tgt_pool_tracker *local_rpt;
+
 		if (myrank != pool->sp_iv_ns->iv_master_rank) {
 			/* If master has been changed, then let's skip
 			 * iv sync, and the new leader will take over
@@ -1817,6 +1820,14 @@ rebuild_task_ult(void *arg)
 		}
 
 		rebuild_leader_status_notify(rgt, pool, task->dst_rebuild_op, myrank);
+
+		local_rpt = rpt_lookup(pool->sp_uuid, task->dst_rebuild_op, rgt->rgt_rebuild_ver,
+				       rgt->rgt_rebuild_gen);
+		if (local_rpt) {
+			local_rpt->rt_abort = 1;
+			D_INFO(DF_RB " set rt_abort", DP_RB_RPT(local_rpt));
+			rpt_put(local_rpt);
+		}
 	}
 
 try_reschedule:
@@ -2385,8 +2396,7 @@ rebuild_tgt_fini(struct rebuild_tgt_pool_tracker *rpt)
 	/* destroy the migrate_tls of 0-xstream */
 	ds_migrate_stop(rpt->rt_pool, rpt->rt_rebuild_ver, rpt->rt_rebuild_gen);
 	/* No one should access rpt after rebuild_fini_one. */
-	D_INFO("Finalized rebuild for "DF_UUID", map_ver=%u.\n",
-	       DP_UUID(rpt->rt_pool_uuid), rpt->rt_rebuild_ver);
+	DL_INFO(rc, DF_RB " Finalized rebuild", DP_RB_RPT(rpt));
 	rpt_delete(rpt);
 }
 
@@ -2515,8 +2525,12 @@ rebuild_tgt_status_check_ult(void *arg)
 				 * it can not find the IV see crt_iv_hdlr_xx().
 				 * let's just stop the rebuild.
 				 */
-				if (rc == -DER_NONEXIST && !status.rebuilding)
+				if (rc == -DER_NONEXIST && !status.rebuilding) {
+					D_INFO(DF_RB ", rc %d, status.rebuilding %d, "
+						     "set rt_global_done",
+					       DP_RB_RPT(rpt), rc, status.rebuilding);
 					rpt->rt_global_done = 1;
+				}
 
 				if (ns->iv_stop) {
 					D_DEBUG(DB_REBUILD, "abort rebuild "

utils/rpms/daos.spec

@@ -10,8 +10,8 @@
 %else
 %global daos_build_args client test
 %endif
-%global mercury_version   2.4.1-2
-%global mercury_version_next   2.4.1-3
+%global mercury_version   2.4.1
+%global mercury_version_max   2.4.1-2
 %global libfabric_version 1.20
 %global argobots_version 1.2-3
 %global __python %{__python3}
@@ -38,7 +38,7 @@ BuildRequires: scons >= 2.4
 %endif
 BuildRequires: libfabric-devel >= %{libfabric_version}
 BuildRequires: mercury-devel >= %{mercury_version}
-BuildRequires: mercury-devel < %{mercury_version_next}
+BuildRequires: mercury-devel <= %{mercury_version_max}
 BuildRequires: gcc-c++
 %if (0%{?rhel} >= 8)
 %global openmpi openmpi
@@ -138,6 +138,7 @@ Requires: mercury-libfabric >= %{mercury_version}
 Requires: mercury-libfabric < %{mercury_version_next}
 
 
+
 %description
 The Distributed Asynchronous Object Storage (DAOS) is an open-source
 software-defined object store designed from the ground up for