issue: 1050049 Move poll_os logic to the internal thread

Fix Alex's comments

Signed-off-by: Liran Oz <lirano@mellanox.com>

Author: Liran Oz

src/vma/Makefile.am

@@ -199,7 +199,6 @@ libvma_la_SOURCES := \
 	event/event_handler_ibverbs.h \
 	event/event_handler_manager.h \
 	event/event_handler_rdma_cm.h \
-	event/event_handler_poll_os.h \
 	event/netlink_event.h \
 	event/route_net_dev_event.h \
 	event/timer_handler.h \

src/vma/event/event_handler_manager.cpp

@@ -37,13 +37,13 @@
 #include <rdma/rdma_cma.h>
 #include <vma/dev/net_device_table_mgr.h>
 #include "vma/dev/ring_allocation_logic.h"
+#include "vma/sock/fd_collection.h"
 #include "vma/sock/sock-redirect.h" // calling orig_os_api.epoll()
 #include "vma/util/verbs_extra.h"
 
 #include "timer_handler.h"
 #include "event_handler_ibverbs.h"
 #include "event_handler_rdma_cm.h"
-#include "event_handler_poll_os.h"
 
 #include "vma/util/instrumentation.h"
 
@@ -178,28 +178,6 @@ void event_handler_manager::unregister_ibverbs_event(int fd, event_handler_ibver
 	post_new_reg_action(reg_action);
 }
 
-void event_handler_manager::register_poll_os_event(int fd, event_handler_poll_os* poll_os_fd)
-{
-	reg_action_t reg_action;
-	memset(&reg_action, 0, sizeof(reg_action));
-	reg_action.type = REGISTER_POLL_OS;
-	reg_action.info.poll_os.fd = fd;
-	reg_action.info.poll_os.poll_os_fd = poll_os_fd;
-	update_epfd(fd, EPOLL_CTL_ADD, EPOLLIN | EPOLLPRI | EPOLLONESHOT);
-	post_new_reg_action(reg_action);
-}
-
-void event_handler_manager::unregister_poll_os_event(int fd, event_handler_poll_os* poll_os_fd)
-{
-	reg_action_t reg_action;
-	memset(&reg_action, 0, sizeof(reg_action));
-	reg_action.type = UNREGISTER_POLL_OS;
-	reg_action.info.poll_os.fd = fd;
-	reg_action.info.poll_os.poll_os_fd = poll_os_fd;
-	update_epfd(fd, EPOLL_CTL_DEL, EPOLLIN | EPOLLPRI | EPOLLONESHOT);
-	post_new_reg_action(reg_action);
-}
-
 void event_handler_manager::register_rdma_cm_event(int fd, void* id, void* cma_channel, event_handler_rdma_cm* handler)
 {
 	reg_action_t reg_action;
@@ -445,8 +423,6 @@ const char* event_handler_manager::reg_action_str(event_action_type_e reg_action
 	case UNREGISTER_RDMA_CM:				return "UNREGISTER_RDMA_CM";
 	case REGISTER_COMMAND:					return "REGISTER_COMMAND";
 	case UNREGISTER_COMMAND:				return "UNREGISTER_COMMAND";
-	case REGISTER_POLL_OS:					return "REGISTER_POLL_OS";
-	case UNREGISTER_POLL_OS:				return "UNREGISTER_POLL_OS";
 	BULLSEYE_EXCLUDE_BLOCK_START
 	default:						return "UNKNOWN";
 	BULLSEYE_EXCLUDE_BLOCK_END
@@ -656,42 +632,6 @@ void event_handler_manager::priv_register_rdma_cm_events(rdma_cm_reg_info_t& inf
 	}
 }
 
-void event_handler_manager::priv_register_poll_os(command_poll_os_info_t& info)
-{
-	evh_logfunc_entry("fd=%d", info.fd);
-
-	event_handler_map_t::iterator iter_fd = m_event_handler_map.find(info.fd);
-	if (iter_fd == m_event_handler_map.end()) {
-		evh_logdbg("Adding poll_os fd event (fd %d)", info.fd);
-		event_data_t map_value;
-
-		map_value.type = EV_POLL_FD;
-		map_value.poll_os_cmd.fd = info.fd;
-		map_value.poll_os_cmd.poll_os_fd = info.poll_os_fd;
-
-		/* coverity[uninit_use_in_call] */
-		/* cppcheck-suppress uninitStructMember */
-		m_event_handler_map[info.fd] = map_value;
-	}
-}
-
-void event_handler_manager::priv_unregister_poll_os(command_poll_os_info_t& info)
-{
-	evh_logfunc_entry("fd=%d", info.fd);
-
-	event_handler_map_t::iterator iter_fd = m_event_handler_map.find(info.fd);
-	if (iter_fd != m_event_handler_map.end()) {
-		if (iter_fd->second.type != EV_POLL_FD){
-			evh_logerr("fd=%d: is already handling events of different type", info.fd);
-			return;
-		}
-		iter_fd->second.poll_os_cmd.poll_os_fd->event_handler_poll_os_unregister_cb();
-		m_event_handler_map.erase(iter_fd);
-	} else {
-		evh_logdbg("POLL_OS event wasn't found (fd %d)", info.fd);
-	}
-}
-
 void event_handler_manager::priv_unregister_rdma_cm_events(rdma_cm_reg_info_t& info)
 {
 	evh_logfunc_entry("fd=%d, id=%p", info.fd, info.id);
@@ -795,12 +735,6 @@ void event_handler_manager::handle_registration_action(reg_action_t& reg_action)
 	case UNREGISTER_COMMAND:
 		priv_unregister_command_events(reg_action.info.cmd);
 		break;
-	case REGISTER_POLL_OS:
-		priv_register_poll_os(reg_action.info.poll_os);
-		break;
-	case UNREGISTER_POLL_OS:
-		priv_unregister_poll_os(reg_action.info.poll_os);
-		break;
 	case UNREGISTER_TIMERS_AND_DELETE:
 		priv_unregister_all_handler_timers(reg_action.info.timer);
 		delete reg_action.info.timer.handler;
@@ -1021,7 +955,10 @@ void* event_handler_manager::thread_loop()
 
 			event_handler_map_t::iterator i = m_event_handler_map.find(fd);
 			if (i == m_event_handler_map.end()) {
-				evh_logdbg("No event handler (fd=%d)", fd);
+				// No event handler - this is probably a poll_os event!
+				if (!g_p_fd_collection->set_immediate_os_sample(fd)) {
+					evh_logdbg("No event handler (fd=%d)", fd);
+				}
 				continue;
 			}
 
@@ -1042,9 +979,6 @@ void* event_handler_manager::thread_loop()
 			case EV_COMMAND:
 				i->second.command_ev.cmd->execute();
 				break;
-			case EV_POLL_FD:
-				i->second.poll_os_cmd.poll_os_fd->event_handler_poll_os_ready_cb();
-				break;
 			BULLSEYE_EXCLUDE_BLOCK_START
 			default:
 				evh_logerr("Unknow event on fd=%d", fd);

src/vma/event/event_handler_manager.h

@@ -47,7 +47,6 @@
 class timer_handler;
 class event_handler_ibverbs;
 class event_handler_rdma_cm;
-class event_handler_poll_os;
 
 typedef std::map<void* /*event_handler_id*/, event_handler_rdma_cm* /*p_event_handler*/> event_handler_rdma_cm_map_t;
 
@@ -61,9 +60,7 @@ typedef enum {
 	REGISTER_RDMA_CM,
 	UNREGISTER_RDMA_CM,
 	REGISTER_COMMAND,
-	UNREGISTER_COMMAND,
-	REGISTER_POLL_OS,
-	UNREGISTER_POLL_OS
+	UNREGISTER_COMMAND
 } event_action_type_e;
 
 
@@ -118,20 +115,13 @@ struct command_reg_info_t {
 	command* 		cmd;
 };
 
-struct command_poll_os_info_t {
-	int             fd;
-	event_handler_poll_os* poll_os_fd;
-};
-typedef command_poll_os_info_t poll_os_ev_t;
-
 struct reg_action_t {
 	event_action_type_e		type;
 	union {
 		timer_reg_info_t	timer;
 		ibverbs_reg_info_t	ibverbs;
 		rdma_cm_reg_info_t	rdma_cm;
 		command_reg_info_t   	cmd;
-		command_poll_os_info_t  poll_os;
 	} info;
 };
 
@@ -141,7 +131,6 @@ enum ev_type {
 	EV_IBVERBS,
 	EV_RDMA_CM,
 	EV_COMMAND,
-	EV_POLL_FD,
 };
 
 
@@ -150,7 +139,6 @@ struct event_data_t {
 	ibverbs_ev_t ibverbs_ev;
 	rdma_cm_ev_t rdma_cm_ev;
 	command_ev_t command_ev;
-	poll_os_ev_t poll_os_cmd;
 };
 
 typedef std::map<int /*fd*/, event_data_t> event_handler_map_t;
@@ -180,9 +168,6 @@ class event_handler_manager : public wakeup_pipe
 	void 	register_rdma_cm_event(int fd, void* id, void* cma_channel, event_handler_rdma_cm* handler);
 	void 	unregister_rdma_cm_event(int fd, void* id);
 
-	void 	register_poll_os_event(int fd, event_handler_poll_os* poll_os_fd);
-	void 	unregister_poll_os_event(int fd, event_handler_poll_os* poll_os_fd);
-
 	void 	register_command_event(int fd, command* cmd);
 	void 	unregister_command_event(int fd);
 
@@ -195,7 +180,7 @@ class event_handler_manager : public wakeup_pipe
 	pthread_t		m_event_handler_tid;
 	bool			m_b_continue_running;
 	int 			m_cq_epfd;
-	int			m_epfd;
+	int 			m_epfd;
 
 	// pipe for the event registration handling
 	reg_action_q_t		m_reg_action_q;
@@ -217,8 +202,6 @@ class event_handler_manager : public wakeup_pipe
 	void	priv_register_rdma_cm_events(rdma_cm_reg_info_t& info);
 	void	priv_unregister_rdma_cm_events(rdma_cm_reg_info_t& info);
 	void 	priv_register_command_events(command_reg_info_t& info);
-	void 	priv_register_poll_os(command_poll_os_info_t& info);
-	void 	priv_unregister_poll_os(command_poll_os_info_t& info);
 	void 	priv_unregister_command_events(command_reg_info_t& info);
 	void	priv_prepare_ibverbs_async_event_queue(event_handler_map_t::iterator& i);
 

src/vma/event/event_handler_poll_os.h

@@ -144,8 +144,9 @@ void fd_collection::clear()
 	}
 
 	//internal thread should be already dead and these sockets should have been deleted through the internal thread.
-	while (!m_pendig_to_remove_lst.empty()) {
-		m_pendig_to_remove_lst.get_and_pop_front()->force_close();
+	list_iterator_t<socket_fd_api, socket_fd_api::pendig_to_remove_node_offset> itr;
+	for (itr = m_pendig_to_remove_lst.begin(); itr != m_pendig_to_remove_lst.end(); itr++) {
+		(*itr)->force_close();
 	}
 
 	// Clean up all left overs sockinfo

src/vma/sock/fd_collection.cpp

@@ -122,6 +122,11 @@ class fd_collection : private lock_mutex_recursive, public timer_handler
 	 */
 	int			del_cq_channel_fd(int fd, bool b_cleanup = false);
 
+	/**
+	 * Call set_immediate_os_sample of the input fd.
+	 */
+	inline bool set_immediate_os_sample(int fd);
+	
 	/**
 	 * Get sock_fd_api (sockinfo or pipeinfo) by fd.
 	 */
@@ -214,6 +219,20 @@ inline cls* fd_collection::get(int fd, cls **map_type)
 	return obj;
 }
 
+inline bool fd_collection::set_immediate_os_sample(int fd)
+{
+	bool ret = false;
+	socket_fd_api* sock;
+
+	lock();
+	if ((sock = get_sockfd(fd))){
+		sock->set_immediate_os_sample();
+		ret = true;
+	}
+	unlock();
+	return ret;
+}
+
 inline socket_fd_api* fd_collection::get_sockfd(int fd)
 {
 	return get(fd, m_p_sockfd_map);