/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2013 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017 Intel, Inc. All rights reserved * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PML_CM_SENDREQ_H #define PML_CM_SENDREQ_H #include "pml_cm_request.h" #include "ompi/mca/pml/base/pml_base_sendreq.h" #include "ompi/mca/pml/base/pml_base_bsend.h" #include "ompi/mca/pml/pml.h" #include "ompi/mca/mtl/mtl.h" #include "opal/prefetch.h" struct mca_pml_cm_send_request_t { mca_pml_cm_request_t req_base; mca_pml_base_send_mode_t req_send_mode; }; typedef struct mca_pml_cm_send_request_t mca_pml_cm_send_request_t; OBJ_CLASS_DECLARATION(mca_pml_cm_send_request_t); struct mca_pml_cm_thin_send_request_t { mca_pml_cm_send_request_t req_send; mca_mtl_request_t req_mtl; /**< the mtl specific memory. This field should be the last in the struct */ }; typedef struct mca_pml_cm_thin_send_request_t mca_pml_cm_thin_send_request_t; OBJ_CLASS_DECLARATION(mca_pml_cm_thin_send_request_t); struct mca_pml_cm_hvy_send_request_t { mca_pml_cm_send_request_t req_send; const void *req_addr; /**< pointer to application buffer */ size_t req_count; /**< count of user datatype elements */ int32_t req_peer; /**< peer process - rank w/in this communicator */ int32_t req_tag; /**< user defined tag */ void *req_buff; /**< pointer to send buffer - may not be application buffer */ bool req_blocking; mca_mtl_request_t req_mtl; /**< the mtl specific memory. This field should be the last in the struct */ }; typedef struct mca_pml_cm_hvy_send_request_t mca_pml_cm_hvy_send_request_t; OBJ_CLASS_DECLARATION(mca_pml_cm_hvy_send_request_t); #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT #define MCA_PML_CM_THIN_SEND_REQUEST_ALLOC(sendreq, comm, dst, \ ompi_proc) \ do { \ ompi_proc = ompi_comm_peer_lookup( comm, dst ); \ \ if(OPAL_UNLIKELY(NULL == ompi_proc)) { \ sendreq = NULL; \ } else { \ sendreq = (mca_pml_cm_thin_send_request_t*) \ opal_free_list_wait (&mca_pml_base_send_requests); \ sendreq->req_send.req_base.req_pml_type = MCA_PML_CM_REQUEST_SEND_THIN; \ sendreq->req_mtl.ompi_req = (ompi_request_t*) sendreq; \ sendreq->req_mtl.completion_callback = mca_pml_cm_send_request_completion; \ } \ } while(0) #else #define MCA_PML_CM_THIN_SEND_REQUEST_ALLOC(sendreq, comm, dst, \ ompi_proc) \ do { \ sendreq = (mca_pml_cm_thin_send_request_t*) \ opal_free_list_wait (&mca_pml_base_send_requests); \ sendreq->req_send.req_base.req_pml_type = MCA_PML_CM_REQUEST_SEND_THIN; \ sendreq->req_mtl.ompi_req = (ompi_request_t*) sendreq; \ sendreq->req_mtl.completion_callback = mca_pml_cm_send_request_completion; \ } while(0) #endif #if (OPAL_ENABLE_HETEROGENEOUS_SUPPORT) #define MCA_PML_CM_HVY_SEND_REQUEST_ALLOC(sendreq, comm, dst, \ ompi_proc) \ { \ ompi_proc = ompi_comm_peer_lookup( comm, dst ); \ if(OPAL_UNLIKELY(NULL == ompi_proc)) { \ sendreq = NULL; \ } else { \ sendreq = (mca_pml_cm_hvy_send_request_t*) \ opal_free_list_wait (&mca_pml_base_send_requests); \ sendreq->req_send.req_base.req_pml_type = MCA_PML_CM_REQUEST_SEND_HEAVY; \ sendreq->req_mtl.ompi_req = (ompi_request_t*) sendreq; \ sendreq->req_mtl.completion_callback = mca_pml_cm_send_request_completion; \ } \ } #else #define MCA_PML_CM_HVY_SEND_REQUEST_ALLOC(sendreq, comm, dst, \ ompi_proc) \ { \ sendreq = (mca_pml_cm_hvy_send_request_t*) \ opal_free_list_wait (&mca_pml_base_send_requests); \ sendreq->req_send.req_base.req_pml_type = MCA_PML_CM_REQUEST_SEND_HEAVY; \ sendreq->req_mtl.ompi_req = (ompi_request_t*) sendreq; \ sendreq->req_mtl.completion_callback = mca_pml_cm_send_request_completion; \ } #endif #if (OPAL_ENABLE_HETEROGENEOUS_SUPPORT) #define MCA_PML_CM_HVY_SEND_REQUEST_INIT_COMMON(req_send, \ ompi_proc, \ comm, \ tag, \ datatype, \ sendmode, \ buf, \ count, \ flags ) \ { \ OBJ_RETAIN(comm); \ OMPI_DATATYPE_RETAIN(datatype); \ (req_send)->req_base.req_comm = comm; \ (req_send)->req_base.req_datatype = datatype; \ MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count); \ opal_convertor_copy_and_prepare_for_send( \ ompi_proc->super.proc_convertor, \ &(datatype->super), \ count, \ buf, \ flags, \ &(req_send)->req_base.req_convertor ); \ (req_send)->req_base.req_ompi.req_mpi_object.comm = comm; \ (req_send)->req_base.req_ompi.req_status.MPI_SOURCE = \ comm->c_my_rank; \ (req_send)->req_base.req_ompi.req_status.MPI_TAG = tag; \ (req_send)->req_base.req_ompi.req_status._ucount = count; \ (req_send)->req_send_mode = sendmode; \ (req_send)->req_base.req_free_called = false; \ } #else #define MCA_PML_CM_HVY_SEND_REQUEST_INIT_COMMON(req_send, \ ompi_proc, \ comm, \ tag, \ datatype, \ sendmode, \ buf, \ count, \ flags ) \ { \ OBJ_RETAIN(comm); \ OMPI_DATATYPE_RETAIN(datatype); \ (req_send)->req_base.req_comm = comm; \ (req_send)->req_base.req_datatype = datatype; \ MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count); \ opal_convertor_copy_and_prepare_for_send( \ ompi_mpi_local_convertor, \ &(datatype->super), \ count, \ buf, \ flags, \ &(req_send)->req_base.req_convertor ); \ (req_send)->req_base.req_ompi.req_mpi_object.comm = comm; \ (req_send)->req_base.req_ompi.req_status.MPI_SOURCE = \ comm->c_my_rank; \ (req_send)->req_base.req_ompi.req_status.MPI_TAG = tag; \ (req_send)->req_base.req_ompi.req_status._ucount = count; \ (req_send)->req_send_mode = sendmode; \ (req_send)->req_base.req_free_called = false; \ } #endif #if (OPAL_ENABLE_HETEROGENEOUS_SUPPORT) #define MCA_PML_CM_SEND_REQUEST_INIT_COMMON(req_send, \ ompi_proc, \ comm, \ tag, \ datatype, \ sendmode, \ buf, \ count, \ flags ) \ { \ OBJ_RETAIN(comm); \ OMPI_DATATYPE_RETAIN(datatype); \ (req_send)->req_base.req_comm = comm; \ (req_send)->req_base.req_datatype = datatype; \ MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count); \ opal_convertor_copy_and_prepare_for_send( \ ompi_proc->super.proc_convertor, \ &(datatype->super), \ count, \ buf, \ flags, \ &(req_send)->req_base.req_convertor ); \ (req_send)->req_base.req_ompi.req_mpi_object.comm = comm; \ (req_send)->req_base.req_ompi.req_status.MPI_SOURCE = \ comm->c_my_rank; \ (req_send)->req_base.req_ompi.req_status.MPI_TAG = tag; \ (req_send)->req_base.req_ompi.req_status._ucount = count; \ (req_send)->req_send_mode = sendmode; \ (req_send)->req_base.req_free_called = false; \ } #else #define MCA_PML_CM_SEND_REQUEST_INIT_COMMON(req_send, \ ompi_proc, \ comm, \ tag, \ datatype, \ sendmode, \ buf, \ count, \ flags ) \ { \ OBJ_RETAIN(comm); \ OMPI_DATATYPE_RETAIN(datatype); \ (req_send)->req_base.req_comm = comm; \ (req_send)->req_base.req_datatype = datatype; \ if (opal_datatype_is_contiguous_memory_layout(&datatype->super, count)) { \ (req_send)->req_base.req_convertor.remoteArch = \ ompi_mpi_local_convertor->remoteArch; \ (req_send)->req_base.req_convertor.flags = \ ompi_mpi_local_convertor->flags; \ (req_send)->req_base.req_convertor.master = \ ompi_mpi_local_convertor->master; \ (req_send)->req_base.req_convertor.local_size = \ count * datatype->super.size; \ (req_send)->req_base.req_convertor.pBaseBuf = \ (unsigned char*)buf + datatype->super.true_lb; \ (req_send)->req_base.req_convertor.count = count; \ (req_send)->req_base.req_convertor.pDesc = &datatype->super; \ } else { \ MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count); \ opal_convertor_copy_and_prepare_for_send( \ ompi_mpi_local_convertor, \ &(datatype->super), \ count, \ buf, \ flags, \ &(req_send)->req_base.req_convertor ); \ } \ (req_send)->req_base.req_ompi.req_mpi_object.comm = comm; \ (req_send)->req_base.req_ompi.req_status.MPI_SOURCE = \ comm->c_my_rank; \ (req_send)->req_base.req_ompi.req_status.MPI_TAG = tag; \ (req_send)->req_base.req_ompi.req_status._ucount = count; \ (req_send)->req_send_mode = sendmode; \ (req_send)->req_base.req_free_called = false; \ } #endif #define MCA_PML_CM_HVY_SEND_REQUEST_INIT( sendreq, \ ompi_proc, \ comm, \ tag, \ dst, \ datatype, \ sendmode, \ persistent, \ blocking, \ buf, \ count, \ flags ) \ do { \ OMPI_REQUEST_INIT(&(sendreq->req_send.req_base.req_ompi), \ persistent); \ sendreq->req_tag = tag; \ sendreq->req_peer = dst; \ sendreq->req_addr = buf; \ sendreq->req_count = count; \ MCA_PML_CM_HVY_SEND_REQUEST_INIT_COMMON( (&sendreq->req_send), \ ompi_proc, \ comm, \ tag, \ datatype, \ sendmode, \ buf, \ count, \ flags ) \ opal_convertor_get_packed_size( \ &sendreq->req_send.req_base.req_convertor, \ &sendreq->req_count ); \ \ sendreq->req_blocking = blocking; \ sendreq->req_send.req_base.req_pml_complete = \ (persistent ? true:false); \ } while(0) #define MCA_PML_CM_THIN_SEND_REQUEST_INIT( sendreq, \ ompi_proc, \ comm, \ tag, \ dst, \ datatype, \ sendmode, \ buf, \ count, \ flags ) \ do { \ OMPI_REQUEST_INIT(&(sendreq->req_send.req_base.req_ompi), \ false); \ MCA_PML_CM_SEND_REQUEST_INIT_COMMON( (&sendreq->req_send), \ ompi_proc, \ comm, \ tag, \ datatype, \ sendmode, \ buf, \ count, \ flags); \ sendreq->req_send.req_base.req_pml_complete = false; \ } while(0) #define MCA_PML_CM_SEND_REQUEST_START_SETUP(req_send) \ do { \ (req_send)->req_base.req_pml_complete = false; \ (req_send)->req_base.req_ompi.req_complete = REQUEST_PENDING; \ (req_send)->req_base.req_ompi.req_state = \ OMPI_REQUEST_ACTIVE; \ (req_send)->req_base.req_ompi.req_status._cancelled = 0; \ } while (0) #define MCA_PML_CM_THIN_SEND_REQUEST_START(sendreq, \ comm, \ tag, \ dst, \ sendmode, \ blocking, \ ret) \ do { \ MCA_PML_CM_SEND_REQUEST_START_SETUP(&(sendreq)->req_send); \ ret = OMPI_MTL_CALL(isend(ompi_mtl, \ comm, \ dst, \ tag, \ &sendreq->req_send.req_base.req_convertor, \ sendmode, \ blocking, \ &sendreq->req_mtl)); \ } while (0) #define MCA_PML_CM_HVY_SEND_REQUEST_BSEND_ALLOC(sendreq, ret) \ do { \ struct iovec iov; \ unsigned int iov_count; \ size_t max_data; \ \ if(sendreq->req_count > 0) { \ sendreq->req_buff = \ mca_pml_base_bsend_request_alloc_buf(sendreq->req_count); \ if (NULL == sendreq->req_buff) { \ ret = MPI_ERR_BUFFER; \ } else { \ iov.iov_base = (IOVBASE_TYPE*)sendreq->req_buff; \ max_data = iov.iov_len = sendreq->req_count; \ iov_count = 1; \ opal_convertor_pack( &sendreq->req_send.req_base.req_convertor, \ &iov, \ &iov_count, \ &max_data ); \ opal_convertor_prepare_for_send( &sendreq->req_send.req_base.req_convertor, \ &(ompi_mpi_packed.dt.super), \ max_data, sendreq->req_buff ); \ } \ } \ } while(0); #define MCA_PML_CM_HVY_SEND_REQUEST_START(sendreq, ret) \ do { \ ret = OMPI_SUCCESS; \ MCA_PML_CM_SEND_REQUEST_START_SETUP(&(sendreq)->req_send); \ if (sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) { \ MCA_PML_CM_HVY_SEND_REQUEST_BSEND_ALLOC(sendreq, ret); \ } \ if (OMPI_SUCCESS == ret) { \ ret = OMPI_MTL_CALL(isend(ompi_mtl, \ sendreq->req_send.req_base.req_comm, \ sendreq->req_peer, \ sendreq->req_tag, \ &sendreq->req_send.req_base.req_convertor, \ sendreq->req_send.req_send_mode, \ sendreq->req_blocking, \ &sendreq->req_mtl)); \ if(OMPI_SUCCESS == ret && \ sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) { \ sendreq->req_send.req_base.req_ompi.req_status.MPI_ERROR = 0; \ if(!REQUEST_COMPLETE(&sendreq->req_send.req_base.req_ompi)) { \ /* request may have already been marked complete by the MTL */ \ ompi_request_complete(&(sendreq)->req_send.req_base.req_ompi, true); \ } \ } \ } \ } while (0) /* * The PML has completed a send request. Note that this request * may have been orphaned by the user or have already completed * at the MPI level. * This macro will never be called directly from the upper level, as it should * only be an internal call to the PML. */ #define MCA_PML_CM_HVY_SEND_REQUEST_PML_COMPLETE(sendreq) \ do { \ assert( false == sendreq->req_send.req_base.req_pml_complete ); \ \ if (sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED && \ sendreq->req_count > 0 ) { \ mca_pml_base_bsend_request_free(sendreq->req_buff); \ } \ \ if( !REQUEST_COMPLETE(&sendreq->req_send.req_base.req_ompi)) { \ /* the request may have already been marked complete by the MTL */ \ ompi_request_complete(&(sendreq->req_send.req_base.req_ompi), true); \ } \ sendreq->req_send.req_base.req_pml_complete = true; \ \ if( sendreq->req_send.req_base.req_free_called ) { \ MCA_PML_CM_HVY_SEND_REQUEST_RETURN( sendreq ); \ } else { \ if(sendreq->req_send.req_base.req_ompi.req_persistent) { \ /* rewind convertor */ \ size_t offset = 0; \ opal_convertor_set_position(&sendreq->req_send.req_base.req_convertor, \ &offset); \ } \ } \ } while (0) /* * Release resources associated with a request */ #define MCA_PML_CM_HVY_SEND_REQUEST_RETURN(sendreq) \ { \ /* Let the base handle the reference counts */ \ OMPI_DATATYPE_RETAIN(sendreq->req_send.req_base.req_datatype); \ OBJ_RELEASE(sendreq->req_send.req_base.req_comm); \ OMPI_REQUEST_FINI(&sendreq->req_send.req_base.req_ompi); \ opal_convertor_cleanup( &(sendreq->req_send.req_base.req_convertor) ); \ opal_free_list_return ( &mca_pml_base_send_requests, \ (opal_free_list_item_t*)sendreq); \ } /* * The PML has completed a send request. Note that this request * may have been orphaned by the user or have already completed * at the MPI level. * This macro will never be called directly from the upper level, as it should * only be an internal call to the PML. */ #define MCA_PML_CM_THIN_SEND_REQUEST_PML_COMPLETE(sendreq) \ do { \ assert( false == sendreq->req_send.req_base.req_pml_complete ); \ \ if( !REQUEST_COMPLETE(&sendreq->req_send.req_base.req_ompi)) { \ /* Should only be called for long messages (maybe synchronous) */ \ ompi_request_complete(&(sendreq->req_send.req_base.req_ompi), true); \ } \ sendreq->req_send.req_base.req_pml_complete = true; \ \ if( sendreq->req_send.req_base.req_free_called ) { \ MCA_PML_CM_THIN_SEND_REQUEST_RETURN( sendreq ); \ } \ } while (0) /* * Release resources associated with a request */ #define MCA_PML_CM_THIN_SEND_REQUEST_RETURN(sendreq) \ { \ /* Let the base handle the reference counts */ \ OMPI_DATATYPE_RETAIN(sendreq->req_send.req_base.req_datatype); \ OBJ_RELEASE(sendreq->req_send.req_base.req_comm); \ OMPI_REQUEST_FINI(&sendreq->req_send.req_base.req_ompi); \ opal_convertor_cleanup( &(sendreq->req_send.req_base.req_convertor) ); \ opal_free_list_return ( &mca_pml_base_send_requests, \ (opal_free_list_item_t*)sendreq); \ } extern void mca_pml_cm_send_request_completion(struct mca_mtl_request_t *mtl_request); #endif