Enable TCP keepalive during connreq processing, otherwise

if the remote peer is restarted between getting connreq and
replying, it may hang there to wait for the connreq reply.
Since the ep state(req_done) will not allow sending new reqs,
and socket state will not be reset, i.e. progress will not be
able to detect the disconnection.

Signed-off-by: Di Wang <[email protected]>

Author: wangdi1

prov/tcp/Makefile.include

@@ -35,4 +35,8 @@ prov_install_man_pages += man/man7/fi_tcp.7
 
 endif HAVE_TCP
 
+if MACOS
+AM_CPPFLAGS += -DMACOS
+endif MACOS
+
 prov_dist_man_pages += man/man7/fi_tcp.7

prov/tcp/src/xnet.h

@@ -781,4 +781,5 @@ int xnet_rdm_ops_open(struct fid *fid, const char *name,
 	FI_WARN(&xnet_prov, subsystem, log_str "%s (%d)\n", \
 		fi_strerror((int) -(err)), (int) err)
 
+int xnet_disable_keepalive(struct xnet_ep *ep);
 #endif //_XNET_H_

prov/tcp/src/xnet_cm.c

@@ -203,9 +203,12 @@ void xnet_req_done(struct xnet_ep *ep)
 	ret = xnet_req_done_internal(ep);
 	if (ret)
 		goto disable;
+
+	xnet_disable_keepalive(ep);
 	return;
 
 disable:
+	xnet_disable_keepalive(ep);
 	xnet_ep_disable(ep, -ret, ep->cm_msg->data,
 			ntohs(ep->cm_msg->hdr.seg_size));
 }

prov/tcp/src/xnet_ep.c

@@ -156,6 +156,81 @@ static void xnet_set_no_port(SOCKET sock)
 #define xnet_set_no_port(sock)
 #endif
 
+int
+xnet_disable_keepalive(struct xnet_ep *ep)
+{
+	int optval = 0;
+	int ret;
+
+	ret = setsockopt(ep->bsock.sock, SOL_SOCKET, SO_KEEPALIVE, (char *)&optval,
+			 sizeof(optval));
+	if (ret) {
+		FI_WARN(&xnet_prov, FI_LOG_EP_CTRL, "set SO_KEEPALIVE failed %d", ret);
+		return -ofi_sockerr();
+	}
+
+	FI_INFO(&xnet_prov, FI_LOG_EP_CTRL, "ep %p KEEPALIVE is disabled.\n", ep);
+	return ret;
+}
+
+static int
+xnet_enable_keepalive(struct xnet_ep *ep)
+{
+	int optval = 1;
+	int idle_time = 5;
+	int keep_intvl = 2;
+	int keep_cnt = 2;
+	int ret;
+
+	ret = setsockopt(ep->bsock.sock, SOL_SOCKET, SO_KEEPALIVE, (const void *)&optval,
+			 sizeof(optval));
+	if (ret) {
+		FI_WARN(&xnet_prov, FI_LOG_EP_CTRL, "set SO_KEEPALIVE failed %d", ret);
+		return -ofi_sockerr();
+	}
+
+#ifdef MACOS
+	ret = setsockopt(ep->bsock.sock, IPPROTO_TCP, TCP_KEEPALIVE, (const void *)&idle_time,
+			 sizeof(idle_time));
+#else
+	ret = setsockopt(ep->bsock.sock, IPPROTO_TCP, TCP_KEEPIDLE, (const void *)&idle_time,
+			 sizeof(idle_time));
+#endif
+	if (ret) {
+		ret = -ofi_sockerr();
+		FI_WARN(&xnet_prov, FI_LOG_EP_CTRL, "set TCP_KEEPIDLE failed %d", ret);
+		goto out;
+	}
+
+	ret = setsockopt(ep->bsock.sock, IPPROTO_TCP, TCP_KEEPINTVL, (const void *)&keep_intvl,
+			 sizeof(keep_intvl));
+	if (ret) {
+		ret = -ofi_sockerr();
+		FI_WARN(&xnet_prov, FI_LOG_EP_CTRL, "set TCP_KEEPINTVL failed %d", ret);
+		goto out;
+	}
+
+	ret = setsockopt(ep->bsock.sock, IPPROTO_TCP, TCP_KEEPCNT, (const void *)&keep_cnt,
+			 sizeof(keep_cnt));
+	if (ret) {
+		ret = -ofi_sockerr();
+		FI_WARN(&xnet_prov, FI_LOG_EP_CTRL, "set SO_KEEPALIVE failed %d", ret);
+		goto out;
+	}
+
+	FI_INFO(&xnet_prov, FI_LOG_EP_CTRL, "%p KEEPALIVE idle %d intvl %d cnt %d\n",
+		ep, idle_time, keep_intvl, keep_cnt);
+
+out:
+	if (ret) {
+		FI_WARN(&xnet_prov, FI_LOG_EP_CTRL, "%p KEEPALIVE set keepalive failed %d\n",
+			ep, ret);
+		xnet_disable_keepalive(ep);
+	}
+
+	return ret;
+}
+
 int xnet_setup_socket(SOCKET sock, struct fi_info *info)
 {
 	int ret, optval = 1;
@@ -294,6 +369,16 @@ xnet_ep_accept(struct fid_ep *ep_fid, const void *param, size_t paramlen)
 		ep->cm_msg->hdr.seg_size = htons((uint16_t) paramlen);
 	}
 
+	/* Enable keepalive to make sure the socket status can be reset in time
+	 * if the remote peer is restarted after it gets connreq but not replies.
+	 */
+	ret = xnet_enable_keepalive(ep);
+	if (ret) {
+		FI_WARN(&xnet_prov, FI_LOG_EP_CTRL, "%p set tcp keepalive failure:%d\n",
+			ep, ret);
+		return ofi_sockerr() ? -ofi_sockerr() : -FI_EINVAL;
+	}
+
 	ret = xnet_send_cm_msg(ep);
 	if (ret)
 		return ret;