prov/verbs: Add support for IBV_ACCESS_RELAXED_ORDERING

IBV_ACCESS_RELAXED_ORDERING allows the system to reorder Send/Write/Atomic operations to improve performance. The patch enables IBV_ACCESS_RELAXED_ORDERING if the application has requested no ordering in TX/RX attributes. Signed-off-by: Sylvain Didelot <sdidelot@ddn.com>
ofiwg · Sep 28, 2023 · c3bc905 · c3bc905
1 parent 7f8a90a
commit c3bc905
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 0 deletions.
diff --git a/prov/verbs/configure.m4 b/prov/verbs/configure.m4
@@ -93,6 +93,16 @@ AC_DEFUN([FI_VERBS_CONFIGURE],[
 	AC_DEFINE_UNQUOTED([VERBS_HAVE_DMABUF_MR],[$VERBS_HAVE_DMABUF_MR],
 		[Whether infiniband/verbs.h has ibv_reg_dmabuf_mr() support or not])
 
+	#See if we have rdma-core IBV_ACCESS_RELAXED_ORDERING mr support
+	VERBS_HAVE_RELAXED_ORDERING_MR=0
+	AS_IF([test $verbs_ibverbs_happy -eq 1],[
+		AC_CHECK_DECL([IBV_ACCESS_RELAXED_ORDERING],
+			[VERBS_HAVE_RELAXED_ORDERING_MR=1],[],
+			[#include <infiniband/verbs.h>])
+		])
+	AC_DEFINE_UNQUOTED([VERBS_HAVE_RELAXED_ORDERING_MR],[$VERBS_HAVE_RELAXED_ORDERING_MR],
+		[Whether infiniband/verbs.h has IBV_ACCESS_RELAXED_ORDERING support or not])
+
 	CPPFLAGS=$fi_verbs_configure_save_CPPFLAGS
 
 	# Technically, verbs_ibverbs_CPPFLAGS and

diff --git a/prov/verbs/src/verbs_mr.c b/prov/verbs/src/verbs_mr.c
@@ -201,6 +201,11 @@ vrb_mr_ofi2ibv_access(uint64_t ofi_access, struct vrb_domain *domain)
 			      IBV_ACCESS_REMOTE_WRITE |
 			      IBV_ACCESS_REMOTE_ATOMIC;
 
+	/* No strict ordering requested */
+	if (domain->util_domain.tx_msg_order == FI_ORDER_NONE &&
+	    domain->util_domain.rx_msg_order == FI_ORDER_NONE)
+		ibv_access |= VRB_ACCESS_RELAXED_ORDERING;
+
 	return ibv_access;
 }
 

diff --git a/prov/verbs/src/verbs_ofi.h b/prov/verbs/src/verbs_ofi.h
@@ -437,6 +437,12 @@ int vrb_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
 		   struct fid_cq **cq, void *context);
 int vrb_cq_trywait(struct vrb_cq *cq);
 
+#if VERBS_HAVE_RELAXED_ORDERING_MR
+#define VRB_ACCESS_RELAXED_ORDERING IBV_ACCESS_RELAXED_ORDERING
+#else
+#define VRB_ACCESS_RELAXED_ORDERING 0
+#endif
+
 struct vrb_mem_desc {
 	struct fid_mr		mr_fid;
 	struct ibv_mr		*mr;