Merge pull request #6409 from raffenet/4.1.x-fixes

4.1.x fixes

Approved-by: Hui Zhou <hzhou321@anl.gov>
Approved-by: Yanfei Guo <yguo@anl.gov>

CHANGES

@@ -1,3 +1,32 @@
+===============================================================================
+                               Changes in 4.1.1
+===============================================================================
+
+# Update embedded UCX module to 1.13.1. Fixes a build issue with
+  binutils >= 2.39.
+
+# Update yaksa module. Support explicit NVCC setting by the user. Fixes
+  a build issue when there is no libtool available in PATH.
+
+# Fix ch4:ucx initialization when configured with
+  --enable-ch4-vci-method=implicit.
+
+# Fix potential error handler leak during MPI_SESSION_FINALIZE
+
+# Fix value of MPI_UNDEFINED in mpif.h binding
+
+# Fix MPI_IALLTOALLW with MPI_IN_PLACE
+
+# Fix send attribute handling in IPC path
+
+# Fix a bug in persistent MPI_ALLGATHER
+
+# Fix tests for use with non-MPICH libraries
+
+# Add missing MPI_T_ERR_NOT_ACCESSIBLE error code
+
+# Fix manpages for MPIX functions
+
 ===============================================================================
                                Changes in 4.1
 ===============================================================================

maint/local_python/binding_c.py

@@ -81,10 +81,8 @@ def dump_romio_reference(name):
         dump_function_internal(func, kind="normal")
     G.out.append("")
 
-    # NOTE: dump_manpage is now called inside dump_qmpi_wrappers
-
     # Create the MPI and QMPI wrapper functions that will call the above, "real" version of the
-    # function in the MPII prefix
+    # function in the internal prefix
     dump_qmpi_wrappers(func, func['_is_large'])
 
 def get_func_file_path(func, root_dir):
@@ -868,6 +866,9 @@ def dump_description(s):
         if l > 0:
             out.append('  ' + ' '.join(words[i0:]))
     # ----
+    if not func['desc']:
+        # place holder to make the man page render
+        func['desc'] = "[short description]"
     out.append("/*D")
     out.append("   %s - %s" % (get_function_name(func, False), func['desc']))
     out.append("")

maint/local_python/binding_f77.py

@@ -438,7 +438,11 @@ def dump_index_out(v):
         c_arg_list_B.append("&%s_i" % v)
         code_list_common.append("int %s_i;" % v)
         end_list_common.append("if (*ierr == MPI_SUCCESS) {")
-        end_list_common.append("    *%s = %s_i + 1;" % (v, v))
+        end_list_common.append("    if (%s_i == MPI_UNDEFINED) {" % v)
+        end_list_common.append("        *%s = %s_i;" % (v, v))
+        end_list_common.append("    } else {")
+        end_list_common.append("        *%s = %s_i + 1;" % (v, v))
+        end_list_common.append("    }")
         end_list_common.append("}")
 
     def dump_string_len_inout(v):

maint/version.m4

@@ -14,7 +14,7 @@
 # changing this by playing with diversions, but then we would probably be
 # playing with autotools-fire.
 
-m4_define([MPICH_VERSION_m4],[4.1])dnl
+m4_define([MPICH_VERSION_m4],[4.1.1])dnl
 m4_define([MPICH_RELEASE_DATE_m4],[unreleased development copy])dnl
 
 # For libtool ABI versioning rules see:
@@ -36,7 +36,7 @@ m4_define([MPICH_RELEASE_DATE_m4],[unreleased development copy])dnl
 # standard, and does not include MPIX_ functions and C++ bindings.
 
 # Use [0:0:0] for unstable (e.g. alpha and beta) releases.
-# last version: 4.1 - 14:4:2
+# last version: 4.1.1 - 15:0:3
 
 m4_define([libmpi_so_version_m4],[0:0:0])dnl
 

src/binding/c/stream_api.txt

@@ -1,39 +1,48 @@
 # vim: set ft=c:
 
 MPIX_Stream_create:
+    .desc: Create a new stream object
     info: INFO, [info argument]
     stream: STREAM, direction=out, [stream object created]
 
 MPIX_Stream_free:
+    .desc: Free a stream object
     stream: STREAM, direction=inout, [stream object]
 
 MPIX_Stream_comm_create:
+    .desc: Create a new communicator with local stream attached
     comm: COMMUNICATOR, [communicator]
     stream: STREAM, [stream object]
     newcomm: COMMUNICATOR, direction=out, [new stream-associated communicator]
 
 MPIX_Stream_comm_create_multiplex:
+    .desc: Create a new communicator with multiple local streams attached
     comm: COMMUNICATOR, [communicator]
     count: ARRAY_LENGTH_NNI, [list length]
     array_of_streams: STREAM, length=count, [stream object array]
     newcomm: COMMUNICATOR, direction=out, [new stream-associated communicator]
 
 MPIX_Comm_get_stream:
+    .desc: Get the stream object that is attached to the communicator
     comm: COMMUNICATOR, [communicator]
     idx: INDEX
     stream: STREAM, direction=out, [stream object]
 
 MPIX_Stream_progress:
     stream: STREAM, [stream object]
+    .desc: Invoke progress on the given stream
     .impl: mpid
 
 MPIX_Start_progress_thread:
+    .desc: Start a progress thread that will poll progress on the given stream
     stream: STREAM, [stream object]
 
 MPIX_Stop_progress_thread:
+    .desc: Stop the progress thread that polls progress on the given stream
     stream: STREAM, [stream object]
 
 MPIX_Stream_send:
+    .desc: Send message from a specific local stream to a specific destination stream
     buf: BUFFER, constant=True, [initial address of send buffer]
     count: POLYXFER_NUM_ELEM_NNI, [number of elements in send buffer]
     datatype: DATATYPE, [datatype of each send buffer element]
@@ -67,6 +76,7 @@ MPIX_Stream_send:
 }
 
 MPIX_Stream_isend:
+    .desc: Start a nonblocking send from a specific local stream to a specific remote stream
     buf: BUFFER, asynchronous=True, constant=True, [initial address of send buffer]
     count: POLYXFER_NUM_ELEM_NNI, [number of elements in send buffer]
     datatype: DATATYPE, [datatype of each send buffer element]
@@ -98,6 +108,7 @@ MPIX_Stream_isend:
 }
 
 MPIX_Stream_recv:
+    .desc: Receive a message from a specific source stream to a specific local stream
     buf: BUFFER, direction=out, [initial address of receive buffer]
     count: POLYXFER_NUM_ELEM_NNI, [number of elements in receive buffer]
     datatype: DATATYPE, [datatype of each receive buffer element]
@@ -134,6 +145,7 @@ MPIX_Stream_recv:
 }
 
 MPIX_Stream_irecv:
+    .desc: Start a nonblocking receive from a specific source stream to a specific local stream
     buf: BUFFER, direction=out, asynchronous=True, suppress=f08_intent, [initial address of receive buffer]
     count: POLYXFER_NUM_ELEM_NNI, [number of elements in receive buffer]
     datatype: DATATYPE, [datatype of each receive buffer element]
@@ -159,6 +171,7 @@ MPIX_Stream_irecv:
 }
 
 MPIX_Send_enqueue:
+    .desc: Enqueue a send operation to a GPU stream that is associated with the local stream
     buf: BUFFER, constant=True, [initial address of send buffer]
     count: POLYXFER_NUM_ELEM_NNI, [number of elements in send buffer]
     datatype: DATATYPE, [datatype of each send buffer element]
@@ -169,6 +182,7 @@ MPIX_Send_enqueue:
     .decl: MPIR_Send_enqueue_impl
 
 MPIX_Recv_enqueue:
+    .desc: Enqueue a receive operation to a GPU stream that is associated with the local stream
     buf: BUFFER, direction=out, [initial address of receive buffer]
     count: POLYXFER_NUM_ELEM_NNI, [number of elements in receive buffer]
     datatype: DATATYPE, [datatype of each receive buffer element]
@@ -180,6 +194,7 @@ MPIX_Recv_enqueue:
     .decl: MPIR_Recv_enqueue_impl
 
 MPIX_Isend_enqueue:
+    .desc: Enqueue a nonblocking send operation to a GPU stream that is associated with the local stream
     buf: BUFFER, constant=True, [initial address of send buffer]
     count: POLYXFER_NUM_ELEM_NNI, [number of elements in send buffer]
     datatype: DATATYPE, [datatype of each send buffer element]
@@ -191,6 +206,7 @@ MPIX_Isend_enqueue:
     .decl: MPIR_Isend_enqueue_impl
 
 MPIX_Irecv_enqueue:
+    .desc: Enqueue a nonblocking receive operation to a GPU stream that is associated with the local stream
     buf: BUFFER, direction=out, [initial address of receive buffer]
     count: POLYXFER_NUM_ELEM_NNI, [number of elements in receive buffer]
     datatype: DATATYPE, [datatype of each receive buffer element]
@@ -202,12 +218,14 @@ MPIX_Irecv_enqueue:
     .decl: MPIR_Irecv_enqueue_impl
 
 MPIX_Wait_enqueue:
+    .desc: Enqueue a wait operation to a GPU stream that is associated with the local stream
     request: REQUEST, direction=inout, [request]
     status: STATUS, direction=out
     .impl: mpid
     .decl: MPIR_Wait_enqueue_impl
 
 MPIX_Waitall_enqueue:
+    .desc: Enqueue a waitall operation to a GPU stream that is associated with the local stream
     count: ARRAY_LENGTH_NNI, [lists length]
     array_of_requests: REQUEST, direction=inout, length=count, [array of requests]
     array_of_statuses: STATUS, direction=out, length=*, pointer=False, [array of status objects]
@@ -220,6 +238,7 @@ MPIX_Waitall_enqueue:
 }
 
 MPIX_Allreduce_enqueue:
+    .desc: Enqueue an allreduce operation to a GPU stream that is associated with the local stream
     sendbuf: BUFFER, constant=True, [starting address of send buffer]
     recvbuf: BUFFER, direction=out, [starting address of receive buffer]
     count: POLYXFER_NUM_ELEM_NNI, [number of elements in send buffer]

src/include/mpi.h.in

@@ -920,10 +920,11 @@ typedef int (MPIX_Grequest_wait_function)(int, void **, double, MPI_Status *);
 #define MPI_ERR_VALUE_TOO_LARGE    77  /* Value is too large to store */
 
 #define MPI_T_ERR_NOT_SUPPORTED    78  /* Requested functionality not supported */
+#define MPI_T_ERR_NOT_ACCESSIBLE   79  /* Requested functionality not accessible */
 
 #define MPI_ERR_LASTCODE    0x3fffffff  /* Last valid error code for a 
 					   predefined error class */
-#define MPICH_ERR_LAST_CLASS 78     /* It is also helpful to know the
+#define MPICH_ERR_LAST_CLASS 79     /* It is also helpful to know the
 				       last valid class */
 
 #define MPICH_ERR_FIRST_MPIX 100 /* Define a gap here because sock is

src/mpi/coll/iallgather/iallgather_intra_sched_recursive_doubling.c

@@ -8,6 +8,7 @@
 /* helper callbacks and associated state structures */
 struct shared_state {
     int recvtype;
+    int recvcount;
     MPI_Aint curr_count;
     MPI_Aint last_recv_count;
     MPI_Status status;
@@ -22,10 +23,11 @@ static int get_count(MPIR_Comm * comm, int tag, void *state)
     return MPI_SUCCESS;
 }
 
-static int dtp_release_ref(MPIR_Comm * comm, int tag, void *state)
+static int reset_shared_state(MPIR_Comm * comm, int tag, void *state)
 {
-    MPIR_Datatype *recv_dtp = state;
-    MPIR_Datatype_ptr_release(recv_dtp);
+    struct shared_state *ss = state;
+    ss->curr_count = ss->recvcount;
+
     return MPI_SUCCESS;
 }
 
@@ -53,7 +55,6 @@ int MPIR_Iallgather_intra_sched_recursive_doubling(const void *sendbuf, MPI_Aint
     int mask, tmp_mask, dst;
     int dst_tree_root, my_tree_root, tree_root;
     MPI_Aint recvtype_extent;
-    MPIR_Datatype *recv_dtp;
 
     comm_size = comm_ptr->local_size;
     rank = comm_ptr->rank;
@@ -64,11 +65,6 @@ int MPIR_Iallgather_intra_sched_recursive_doubling(const void *sendbuf, MPI_Aint
     MPIR_Assert(!(comm_size & (comm_size - 1)));
 #endif /* HAVE_ERROR_CHECKING */
 
-    recv_dtp = NULL;
-    if (!HANDLE_IS_BUILTIN(recvtype)) {
-        MPIR_Datatype_get_ptr(recvtype, recv_dtp);
-    }
-
     MPIR_Datatype_get_extent_macro(recvtype, recvtype_extent);
 
     /*  copy local data into recvbuf */
@@ -84,9 +80,7 @@ int MPIR_Iallgather_intra_sched_recursive_doubling(const void *sendbuf, MPI_Aint
     MPIR_ERR_CHKANDJUMP(!ss, mpi_errno, MPI_ERR_OTHER, "**nomem");
     ss->curr_count = recvcount;
     ss->recvtype = recvtype;
-    /* ensure that recvtype doesn't disappear immediately after last _recv but before _cb */
-    if (recv_dtp)
-        MPIR_Datatype_ptr_add_ref(recv_dtp);
+    ss->recvcount = recvcount;
 
     mask = 0x1;
     i = 0;
@@ -205,10 +199,8 @@ int MPIR_Iallgather_intra_sched_recursive_doubling(const void *sendbuf, MPI_Aint
         i++;
     }
 
-    if (recv_dtp) {
-        mpi_errno = MPIR_Sched_cb(dtp_release_ref, recv_dtp, s);
-        MPIR_ERR_CHECK(mpi_errno);
-    }
+    mpi_errno = MPIR_Sched_cb(reset_shared_state, ss, s);
+    MPIR_ERR_CHECK(mpi_errno);
 
   fn_exit:
     return mpi_errno;

src/mpi/coll/ialltoallw/ialltoallw_intra_sched_inplace.c

@@ -65,7 +65,7 @@ int MPIR_Ialltoallw_intra_sched_inplace(const void *sendbuf, const MPI_Aint send
                 else
                     dst = i;
 
-                MPIR_Datatype_get_size_macro(recvtypes[i], recvtype_sz);
+                MPIR_Datatype_get_size_macro(recvtypes[dst], recvtype_sz);
                 mpi_errno = MPIR_Sched_send(((char *) recvbuf + rdispls[dst]),
                                             recvcounts[dst], recvtypes[dst], dst, comm_ptr, s);
                 MPIR_ERR_CHECK(mpi_errno);

src/mpi/coll/ialltoallw/ialltoallw_tsp_inplace.c

@@ -36,6 +36,10 @@ int MPIR_TSP_Ialltoallw_sched_intra_inplace(const void *sendbuf, const MPI_Aint
     mpi_errno = MPIR_Sched_next_tag(comm, &tag);
     MPIR_ERR_CHECK(mpi_errno);
 
+    /* FIXME: Here we allocate tmp_buf using extent and send/recv with datatype directly,
+     *        which can be potentially very inefficient. Why don't we use bytes as in
+     *        ialltoallw_intra_sched_inplace.c ?
+     */
     MPI_Aint max_size;
     max_size = 0;
     for (i = 0; i < nranks; ++i) {

src/mpi/init/mpir_init.c

@@ -365,6 +365,10 @@ int MPII_Finalize(MPIR_Session * session_ptr)
         MPID_Thread_mutex_destroy(&session_ptr->mutex, &thr_err);
         MPIR_Assert(thr_err == 0);
 
+        if (session_ptr->errhandler != NULL) {
+            MPIR_Errhandler_free_impl(session_ptr->errhandler);
+        }
+
         MPIR_Handle_obj_free(&MPIR_Session_mem, session_ptr);
     }
 

src/mpi/request/request_impl.c

@@ -22,7 +22,9 @@
         MPIR_CVAR_REQUEST_ERR_FATAL is set to true, these routines will
         return the error code of the request immediately. The default
         MPI_ERRS_ARE_FATAL error handler will dump a error stack in this
-        case, which maybe more convenient for debugging.
+        case, which maybe more convenient for debugging. This cvar will also
+        make nonblocking shched return error right away as it issues
+        operations.
 
 === END_MPI_T_CVAR_INFO_BLOCK ===
 */

src/mpid/ch4/shm/ipc/src/ipc_send.h

@@ -133,11 +133,9 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_IPC_mpi_isend(const void *buf, MPI_Aint count
     int mpi_errno = MPI_SUCCESS;
     MPIR_FUNC_ENTER;
 
-    int context_offset = MPIR_PT2PT_ATTR_CONTEXT_OFFSET(attr);
-
     bool done = false;
     mpi_errno = MPIDI_IPCI_try_lmt_isend(buf, count, datatype, rank, tag, comm,
-                                         context_offset, addr, request, &done);
+                                         attr, addr, request, &done);
     MPIR_ERR_CHECK(mpi_errno);
 
     if (!done) {

src/mpid/ch4/src/ch4_vci.h

@@ -129,11 +129,9 @@ static bool is_vci_restricted_to_zero(MPIR_Comm * comm)
     if (!(comm->comm_kind == MPIR_COMM_KIND__INTRACOMM && !comm->tainted)) {
         vci_restricted |= true;
     }
-#ifdef  MPIDI_OFI_VNI_USE_DOMAIN
     if (!MPIDI_global.is_initialized) {
         vci_restricted |= true;
     }
-#endif /* ifdef  MPIDI_OFI_VNI_USE_DOMAIN */
 
     return vci_restricted;
 }