Skip to content

Commit 8c5a66c

Browse files
soumagneknard38
soumagne
authored and
knard38
committed
DAOS-19299 mercury: add patch to fix handling of canceled events
This would have caused RPCs to remain in flight until timeout was reached Quick-Functional: true Test-repeat: 10 Test-tag: DaosCoreTest,test_daos_extend_simple Signed-off-by: Jerome Soumagne <jerome.soumagne@hpe.com>
1 parent d305d2d commit 8c5a66c

4 files changed

Lines changed: 81 additions & 2 deletions

File tree

‎deps/patches/mercury/0006_canceled_events.patch‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
From 061e86519f587308d44e424e02e6b6b208ddafe4 Mon Sep 17 00:00:00 2001
2+
From: Jerome Soumagne <jerome.soumagne@hpe.com>
3+
Date: Mon, 20 Jul 2026 17:14:52 -0500
4+
Subject: [PATCH] HG core: fix handling of canceled event on send unexpected
5+
6+
In rare cases, the underlying provider could return a canceled event
7+
without explicit user cancelation, which would lead to a lingering
8+
recv not canceled (since CANCELED is handled separately from other
9+
error codes). This could have in turn led to a hang until explicit
10+
cancelation was made.
11+
---
12+
src/mercury_core.c | 48 ++++++++++++++++++++++++++--------------------
13+
1 file changed, 27 insertions(+), 21 deletions(-)
14+
15+
diff --git a/src/mercury_core.c b/src/mercury_core.c
16+
index 06404586..eb7d2803 100644
17+
--- a/src/mercury_core.c
18+
+++ b/src/mercury_core.c
19+
@@ -4433,30 +4433,36 @@ hg_core_send_input_cb(const struct na_cb_info *callback_info)
20+
21+
if (callback_info->ret == NA_SUCCESS) {
22+
/* Nothing */
23+
- } else if (callback_info->ret == NA_CANCELED) {
24+
- HG_CHECK_SUBSYS_WARNING(rpc,
25+
- hg_atomic_get32(&hg_core_handle->status) & HG_CORE_OP_COMPLETED,
26+
- "Operation was completed");
27+
- HG_LOG_SUBSYS_DEBUG(
28+
- rpc, "NA_CANCELED event on handle %p", (void *) hg_core_handle);
29+
-
30+
- hg_atomic_cas32(&hg_core_handle->ret_status, (int32_t) HG_SUCCESS,
31+
- (int32_t) HG_CANCELED);
32+
- } else { /* All other errors */
33+
- char addr_buf[HG_CORE_ADDR_MAX_SIZE];
34+
- size_t addr_buf_size = sizeof(addr_buf);
35+
+ } else {
36+
int32_t status;
37+
38+
- /* Mark handle as errored */
39+
- status = hg_atomic_or32(&hg_core_handle->status, HG_CORE_OP_ERRORED);
40+
+ if (callback_info->ret == NA_CANCELED) {
41+
+ status = hg_atomic_get32(&hg_core_handle->status);
42+
43+
- /* Keep first non-success ret status */
44+
- hg_atomic_cas32(&hg_core_handle->ret_status, (int32_t) HG_SUCCESS,
45+
- (int32_t) callback_info->ret);
46+
- HG_LOG_SUBSYS_ERROR(rpc, "NA callback returned error (%s, dest=\"%s\")",
47+
- NA_Error_to_string(callback_info->ret),
48+
- hg_core_na_addr_to_string(hg_core_handle->na_class,
49+
- hg_core_handle->na_addr, addr_buf, &addr_buf_size));
50+
+ HG_CHECK_SUBSYS_WARNING(
51+
+ rpc, status & HG_CORE_OP_COMPLETED, "Operation was completed");
52+
+ HG_LOG_SUBSYS_DEBUG(
53+
+ rpc, "NA_CANCELED event on handle %p", (void *) hg_core_handle);
54+
+
55+
+ hg_atomic_cas32(&hg_core_handle->ret_status, (int32_t) HG_SUCCESS,
56+
+ (int32_t) HG_CANCELED);
57+
+ } else { /* All other errors */
58+
+ char addr_buf[HG_CORE_ADDR_MAX_SIZE];
59+
+ size_t addr_buf_size = sizeof(addr_buf);
60+
+
61+
+ /* Mark handle as errored */
62+
+ status =
63+
+ hg_atomic_or32(&hg_core_handle->status, HG_CORE_OP_ERRORED);
64+
+
65+
+ /* Keep first non-success ret status */
66+
+ hg_atomic_cas32(&hg_core_handle->ret_status, (int32_t) HG_SUCCESS,
67+
+ (int32_t) callback_info->ret);
68+
+ HG_LOG_SUBSYS_ERROR(rpc,
69+
+ "NA callback returned error (%s, dest=\"%s\")",
70+
+ NA_Error_to_string(callback_info->ret),
71+
+ hg_core_na_addr_to_string(hg_core_handle->na_class,
72+
+ hg_core_handle->na_addr, addr_buf, &addr_buf_size));
73+
+ }
74+
75+
if (!(status & HG_CORE_OP_CANCELED) &&
76+
!(hg_atomic_get32(&hg_core_handle->flags) & HG_CORE_NO_RESPONSE)) {

‎utils/build.config‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,6 @@ ucx=https://github.com/openucx/ucx.git
2727

2828
[patch_versions]
2929
spdk=0001_external_isal_v26.01.patch
30-
mercury=0001_dep_versions.patch,0002_ofi_counters.patch,0003_ofi_auth_key.patch,0004_ucx_connection_reset.patch,0005_ucx_put_flush.patch
30+
mercury=0001_dep_versions.patch,0002_ofi_counters.patch,0003_ofi_auth_key.patch,0004_ucx_connection_reset.patch,0005_ucx_put_flush.patch,0006_canceled_events.patch
3131
pmdk=https://github.com/daos-stack/pmdk/commit/bb048d67ccd07609f86a5e8b3c6ad54414d593ee.diff,https://github.com/daos-stack/pmdk/commit/69925cf455ef672c4cbdbdb13bef7ae581e67045.diff,https://github.com/daos-stack/pmdk/commit/6805ed4f8d1a4e4c6070bf8b68f0dffef08b9c99.diff
3232
argobots=0001_411e5b344642ebc82190fd8b125db512e5b449d1.diff,0002_bb0c908abfac4bfe37852eee621930634183c6aa.diff

‎utils/rpms/mercury.changelog‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
%changelog
2+
* Tue Jul 21 2026 Jerome Soumagne <jerome.soumagne@hpe.com> - 2.4.1-4
3+
- Add patch to fix handling of canceled events
4+
25
* Fri May 29 2026 Joseph Moore <joseph.moore@hpe.com> - 2.4.1-3
36
- Add NA UCX patch to prevent reconnection on accepted addresses
47
- Add NA UCX patch to flush RMA puts

‎utils/rpms/package_info.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ export libfabric_version="1.22.0"
4545
export libfabric_release="5${distro_name}"
4646
export libfabric_full="${libfabric_version}-${libfabric_release}"
4747
export mercury_version="2.4.1"
48-
export mercury_release="3${distro_name}"
48+
export mercury_release="4${distro_name}"
4949
export mercury_full="${mercury_version}-${mercury_release}"
5050
export argobots_version="1.2"
5151
export argobots_release="4${distro_name}"

0 commit comments

Comments
 (0)