Introduce Completion Counters verbs

debian/ibverbs-providers.symbols

@@ -200,6 +200,7 @@ libefa.so.1 ibverbs-providers #MINVER#
  efadv_get_max_sq_depth@EFA_1.5 63
  efadv_get_max_rq_depth@EFA_1.5 63
  efadv_qp_from_ibv_qp_ex@EFA_1.6 64
+ efadv_create_comp_cntr@EFA_1.6 64
 libhns.so.1 ibverbs-providers #MINVER#
 * Build-Depends-Package: libibverbs-dev
  HNS_1.0@HNS_1.0 51

debian/libibverbs1.symbols

@@ -14,6 +14,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
  IBVERBS_1.14@IBVERBS_1.14 36
  IBVERBS_1.15@IBVERBS_1.15 59
  IBVERBS_1.16@IBVERBS_1.16 62
+ IBVERBS_1.17@IBVERBS_1.17 64
  (symver)IBVERBS_PRIVATE_59 59
  _ibv_query_gid_ex@IBVERBS_1.11 32
  _ibv_query_gid_table@IBVERBS_1.11 32
@@ -36,6 +37,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
  ibv_create_ah@IBVERBS_1.1 1.1.6
  ibv_create_ah_from_wc@IBVERBS_1.1 1.1.6
  ibv_create_comp_channel@IBVERBS_1.0 1.1.6
+ ibv_create_comp_cntr@IBVERBS_1.17 64
  ibv_create_cq@IBVERBS_1.0 1.1.6
  ibv_create_cq@IBVERBS_1.1 1.1.6
  ibv_create_qp@IBVERBS_1.0 1.1.6
@@ -50,6 +52,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
  ibv_destroy_ah@IBVERBS_1.0 1.1.6
  ibv_destroy_ah@IBVERBS_1.1 1.1.6
  ibv_destroy_comp_channel@IBVERBS_1.0 1.1.6
+ ibv_destroy_comp_cntr@IBVERBS_1.17 64
  ibv_destroy_cq@IBVERBS_1.0 1.1.6
  ibv_destroy_cq@IBVERBS_1.1 1.1.6
  ibv_destroy_qp@IBVERBS_1.0 1.1.6
@@ -82,6 +85,8 @@ libibverbs.so.1 libibverbs1 #MINVER#
  ibv_import_dm@IBVERBS_1.13 35
  ibv_import_mr@IBVERBS_1.10 31
  ibv_import_pd@IBVERBS_1.10 31
+ ibv_inc_comp_cntr@IBVERBS_1.17 64
+ ibv_inc_err_comp_cntr@IBVERBS_1.17 64
  ibv_init_ah_from_wc@IBVERBS_1.1 1.1.6
  ibv_is_fork_initialized@IBVERBS_1.13 35
  ibv_modify_qp@IBVERBS_1.0 1.1.6
@@ -92,6 +97,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
  ibv_open_device@IBVERBS_1.0 1.1.6
  ibv_open_device@IBVERBS_1.1 1.1.6
  ibv_port_state_str@IBVERBS_1.1 1.1.6
+ ibv_qp_attach_comp_cntr@IBVERBS_1.17 64
  ibv_qp_to_qp_ex@IBVERBS_1.6 24
  ibv_query_device@IBVERBS_1.0 1.1.6
  ibv_query_device@IBVERBS_1.1 1.1.6
@@ -110,6 +116,8 @@ libibverbs.so.1 libibverbs1 #MINVER#
  ibv_query_srq@IBVERBS_1.1 1.1.6
  ibv_rate_to_mbps@IBVERBS_1.1 1.1.8
  ibv_rate_to_mult@IBVERBS_1.0 1.1.6
+ ibv_read_comp_cntr@IBVERBS_1.17 64
+ ibv_read_err_comp_cntr@IBVERBS_1.17 64
  ibv_read_sysfs_file@IBVERBS_1.0 1.1.6
  ibv_reg_dmabuf_mr@IBVERBS_1.12 34
  ibv_reg_mr@IBVERBS_1.0 1.1.6
@@ -122,7 +130,9 @@ libibverbs.so.1 libibverbs1 #MINVER#
  ibv_resize_cq@IBVERBS_1.0 1.1.6
  ibv_resize_cq@IBVERBS_1.1 1.1.6
  ibv_resolve_eth_l2_from_gid@IBVERBS_1.1 1.2.0
+ ibv_set_comp_cntr@IBVERBS_1.17 64
  ibv_set_ece@IBVERBS_1.10 31
+ ibv_set_err_comp_cntr@IBVERBS_1.17 64
  ibv_unimport_dm@IBVERBS_1.13 35
  ibv_unimport_mr@IBVERBS_1.10 31
  ibv_unimport_pd@IBVERBS_1.10 31

kernel-headers/rdma/efa-abi.h

@@ -133,6 +133,7 @@ enum {
 	EFA_QUERY_DEVICE_CAPS_RDMA_WRITE = 1 << 5,
 	EFA_QUERY_DEVICE_CAPS_UNSOLICITED_WRITE_RECV = 1 << 6,
 	EFA_QUERY_DEVICE_CAPS_CQ_WITH_EXT_MEM = 1 << 7,
+	EFA_QUERY_DEVICE_CAPS_COMP_CNTR = 1 << 8,
 };
 
 struct efa_ibv_ex_query_device_resp {
@@ -163,4 +164,22 @@ enum efa_mr_methods {
 	EFA_IB_METHOD_MR_QUERY = (1U << UVERBS_ID_NS_SHIFT),
 };
 
+enum efa_uverbs_buffer_type {
+	EFA_UVERBS_BUFFER_TYPE_DMABUF = 0,
+	EFA_UVERBS_BUFFER_TYPE_VA = 1,
+};
+
+struct efa_uverbs_buffer_desc {
+	__s32 fd;
+	__u32 type;
+	__u32 reserved[2];
+	__aligned_u64 addr;
+	__aligned_u64 length;
+};
+
+enum efa_comp_cntr_create_attrs {
+	EFA_IB_ATTR_CREATE_COMP_CNTR_COMP_BUFFER = (1U << UVERBS_ID_NS_SHIFT),
+	EFA_IB_ATTR_CREATE_COMP_CNTR_ERR_BUFFER,
+};
+
 #endif /* EFA_ABI_USER_H */

kernel-headers/rdma/ib_user_ioctl_cmds.h

@@ -57,6 +57,7 @@ enum uverbs_default_objects {
 	UVERBS_OBJECT_ASYNC_EVENT,
 	UVERBS_OBJECT_DMAH,
 	UVERBS_OBJECT_DMABUF,
+	UVERBS_OBJECT_COMP_CNTR,
 };
 
 enum {
@@ -165,9 +166,16 @@ enum uverbs_attrs_destroy_qp_cmd_attr_ids {
 	UVERBS_ATTR_DESTROY_QP_RESP,
 };
 
+enum uverbs_attrs_qp_attach_comp_cntr_cmd_attr_ids {
+	UVERBS_ATTR_QP_ATTACH_COMP_CNTR_HANDLE,
+	UVERBS_ATTR_QP_ATTACH_COMP_CNTR_CNTR_HANDLE,
+	UVERBS_ATTR_QP_ATTACH_COMP_CNTR_OP_MASK,
+};
+
 enum uverbs_methods_qp {
 	UVERBS_METHOD_QP_CREATE,
 	UVERBS_METHOD_QP_DESTROY,
+	UVERBS_METHOD_QP_ATTACH_COMP_CNTR,
 };
 
 enum uverbs_attrs_create_srq_cmd_attr_ids {
@@ -434,4 +442,34 @@ enum uverbs_attrs_query_gid_entry_cmd_attr_ids {
 	UVERBS_ATTR_QUERY_GID_ENTRY_RESP_ENTRY,
 };
 
+enum uverbs_methods_comp_cntr {
+	UVERBS_METHOD_COMP_CNTR_CREATE,
+	UVERBS_METHOD_COMP_CNTR_DESTROY,
+	UVERBS_METHOD_COMP_CNTR_MODIFY,
+	UVERBS_METHOD_COMP_CNTR_READ,
+};
+
+enum uverbs_attrs_create_comp_cntr_cmd_attr_ids {
+	UVERBS_ATTR_CREATE_COMP_CNTR_HANDLE,
+	UVERBS_ATTR_CREATE_COMP_CNTR_RESP_COUNT_MAX_VALUE,
+	UVERBS_ATTR_CREATE_COMP_CNTR_RESP_ERR_COUNT_MAX_VALUE,
+};
+
+enum uverbs_attrs_destroy_comp_cntr_cmd_attr_ids {
+	UVERBS_ATTR_DESTROY_COMP_CNTR_HANDLE,
+};
+
+enum uverbs_attrs_modify_comp_cntr_cmd_attr_ids {
+	UVERBS_ATTR_MODIFY_COMP_CNTR_HANDLE,
+	UVERBS_ATTR_MODIFY_COMP_CNTR_ENTRY,
+	UVERBS_ATTR_MODIFY_COMP_CNTR_OP,
+	UVERBS_ATTR_MODIFY_COMP_CNTR_VALUE,
+};
+
+enum uverbs_attrs_read_comp_cntr_cmd_attr_ids {
+	UVERBS_ATTR_READ_COMP_CNTR_HANDLE,
+	UVERBS_ATTR_READ_COMP_CNTR_ENTRY,
+	UVERBS_ATTR_READ_COMP_CNTR_RESP_VALUE,
+};
+
 #endif

kernel-headers/rdma/ib_user_ioctl_verbs.h

@@ -273,4 +273,23 @@ struct ib_uverbs_gid_entry {
 	__u32 netdev_ifindex; /* It is 0 if there is no netdev associated with it */
 };
 
+enum ib_uverbs_comp_cntr_entry {
+	IB_UVERBS_COMP_CNTR_ENTRY_COMP,
+	IB_UVERBS_COMP_CNTR_ENTRY_ERR,
+};
+
+enum ib_uverbs_comp_cntr_modify_op {
+	IB_UVERBS_COMP_CNTR_MODIFY_OP_SET,
+	IB_UVERBS_COMP_CNTR_MODIFY_OP_INC,
+};
+
+enum ib_uverbs_qp_attach_comp_cntr_op {
+	IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_SEND = 1 << 0,
+	IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RECV = 1 << 1,
+	IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_READ = 1 << 2,
+	IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ = 1 << 3,
+	IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE = 1 << 4,
+	IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE = 1 << 5,
+};
+
 #endif

kernel-headers/rdma/ib_user_verbs.h

@@ -299,7 +299,7 @@ struct ib_uverbs_ex_query_device_resp {
 	struct ib_uverbs_cq_moderation_caps cq_moderation_caps;
 	__aligned_u64 max_dm_size;
 	__u32 xrc_odp_caps;
-	__u32 reserved;
+	__u32 max_comp_cntr;
 };
 
 struct ib_uverbs_query_port {

libibverbs/CMakeLists.txt

@@ -21,11 +21,12 @@ configure_file("libibverbs.map.in"
 
 rdma_library(ibverbs "${CMAKE_CURRENT_BINARY_DIR}/libibverbs.map"
   # See Documentation/versioning.md
-  1 1.16.${PACKAGE_VERSION}
+  1 1.17.${PACKAGE_VERSION}
   all_providers.c
   cmd.c
   cmd_ah.c
   cmd_counters.c
+  cmd_comp_cntr.c
   cmd_cq.c
   cmd_device.c
   cmd_dm.c

libibverbs/cmd_comp_cntr.c

@@ -0,0 +1,131 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#include <infiniband/cmd_write.h>
+
+int ibv_cmd_create_comp_cntr(struct ibv_context *context,
+			     struct ibv_comp_cntr *comp_cntr,
+			     struct ibv_command_buffer *link)
+{
+	DECLARE_COMMAND_BUFFER_LINK(cmdb, UVERBS_OBJECT_COMP_CNTR,
+				    UVERBS_METHOD_COMP_CNTR_CREATE, 3, link);
+	struct ib_uverbs_attr *handle;
+	int ret;
+
+	comp_cntr->context = context;
+
+	handle = fill_attr_out_obj(cmdb, UVERBS_ATTR_CREATE_COMP_CNTR_HANDLE);
+	fill_attr_out_ptr(cmdb, UVERBS_ATTR_CREATE_COMP_CNTR_RESP_COUNT_MAX_VALUE,
+			  &comp_cntr->comp_count_max_value);
+	fill_attr_out_ptr(cmdb, UVERBS_ATTR_CREATE_COMP_CNTR_RESP_ERR_COUNT_MAX_VALUE,
+			  &comp_cntr->err_count_max_value);
+
+	ret = execute_ioctl(context, cmdb);
+	if (ret)
+		return errno;
+
+	comp_cntr->handle = read_attr_obj(UVERBS_ATTR_CREATE_COMP_CNTR_HANDLE, handle);
+	return 0;
+}
+
+int ibv_cmd_destroy_comp_cntr(struct ibv_comp_cntr *comp_cntr)
+{
+	DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_COMP_CNTR, UVERBS_METHOD_COMP_CNTR_DESTROY, 1);
+	int ret;
+
+	fill_attr_in_obj(cmdb, UVERBS_ATTR_DESTROY_COMP_CNTR_HANDLE, comp_cntr->handle);
+
+	ret = execute_ioctl(comp_cntr->context, cmdb);
+	if (verbs_is_destroy_err(&ret))
+		return ret;
+
+	return 0;
+}
+
+static int ibv_icmd_modify_comp_cntr(struct ibv_comp_cntr *comp_cntr,
+				     uint8_t entry, uint8_t op, uint64_t value)
+{
+	DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_COMP_CNTR, UVERBS_METHOD_COMP_CNTR_MODIFY, 4);
+
+	fill_attr_in_obj(cmdb, UVERBS_ATTR_MODIFY_COMP_CNTR_HANDLE, comp_cntr->handle);
+	fill_attr_const_in(cmdb, UVERBS_ATTR_MODIFY_COMP_CNTR_ENTRY, entry);
+	fill_attr_const_in(cmdb, UVERBS_ATTR_MODIFY_COMP_CNTR_OP, op);
+	fill_attr_in_uint64(cmdb, UVERBS_ATTR_MODIFY_COMP_CNTR_VALUE, value);
+
+	return execute_ioctl(comp_cntr->context, cmdb);
+}
+
+int ibv_cmd_set_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value)
+{
+	return ibv_icmd_modify_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_COMP,
+					 IB_UVERBS_COMP_CNTR_MODIFY_OP_SET, value);
+}
+
+int ibv_cmd_set_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value)
+{
+	return ibv_icmd_modify_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_ERR,
+					 IB_UVERBS_COMP_CNTR_MODIFY_OP_SET, value);
+}
+
+int ibv_cmd_inc_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount)
+{
+	return ibv_icmd_modify_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_COMP,
+					 IB_UVERBS_COMP_CNTR_MODIFY_OP_INC, amount);
+}
+
+int ibv_cmd_inc_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount)
+{
+	return ibv_icmd_modify_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_ERR,
+					 IB_UVERBS_COMP_CNTR_MODIFY_OP_INC, amount);
+}
+
+static int ibv_icmd_read_comp_cntr(struct ibv_comp_cntr *comp_cntr,
+				   uint8_t entry, uint64_t *value)
+{
+	DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_COMP_CNTR, UVERBS_METHOD_COMP_CNTR_READ, 3);
+
+	fill_attr_in_obj(cmdb, UVERBS_ATTR_READ_COMP_CNTR_HANDLE, comp_cntr->handle);
+	fill_attr_const_in(cmdb, UVERBS_ATTR_READ_COMP_CNTR_ENTRY, entry);
+	fill_attr_out_ptr(cmdb, UVERBS_ATTR_READ_COMP_CNTR_RESP_VALUE, value);
+
+	return execute_ioctl(comp_cntr->context, cmdb);
+}
+
+int ibv_cmd_read_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value)
+{
+	return ibv_icmd_read_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_COMP, value);
+}
+
+int ibv_cmd_read_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value)
+{
+	return ibv_icmd_read_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_ERR, value);
+}
+
+int ibv_cmd_qp_attach_comp_cntr(struct ibv_qp *qp, struct ibv_comp_cntr *comp_cntr,
+				struct ibv_qp_attach_comp_cntr_attr *attr)
+{
+	DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_QP, UVERBS_METHOD_QP_ATTACH_COMP_CNTR, 3);
+	uint32_t op_mask = 0;
+
+	fill_attr_in_obj(cmdb, UVERBS_ATTR_QP_ATTACH_COMP_CNTR_HANDLE, qp->handle);
+	fill_attr_in_obj(cmdb, UVERBS_ATTR_QP_ATTACH_COMP_CNTR_CNTR_HANDLE, comp_cntr->handle);
+
+	if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_SEND)
+		op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_SEND;
+	if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_RECV)
+		op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RECV;
+	if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_READ)
+		op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_READ;
+	if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ)
+		op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ;
+	if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE)
+		op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE;
+	if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE)
+		op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE;
+
+	fill_attr_in_uint32(cmdb, UVERBS_ATTR_QP_ATTACH_COMP_CNTR_OP_MASK, op_mask);
+
+	return execute_ioctl(qp->context, cmdb);
+}

libibverbs/cmd_device.c

@@ -694,6 +694,10 @@ int ibv_cmd_query_device_any(struct ibv_context *context,
 		else
 			attr->phys_port_cnt_ex = attr->orig_attr.phys_port_cnt;
 	}
+
+	if (CAN_COPY(max_comp_cntr, max_comp_cntr))
+		attr->max_comp_cntr = resp->max_comp_cntr;
+
 #undef CAN_COPY
 
 	return 0;