nm_mpi_private.h

Go to the documentation of this file.

/*
 * NewMadeleine
 * Copyright (C) 2006-2026 (see AUTHORS file)
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 */
 
#ifndef NM_MPI_PRIVATE_H
#define NM_MPI_PRIVATE_H
 
#include <stdint.h>
#include <limits.h>
#include <unistd.h>
#include <assert.h>
#include <complex.h>
#include <sys/stat.h>
 
#include <Padico/Puk.h>
#include <nm_private_config.h>
#include <nm_public.h>
#include <nm_core_interface.h>
#include <nm_sendrecv_interface.h>
#include <nm_pack_interface.h>
#include <nm_launcher_interface.h>
#include <nm_coll_interface.h>
 
#include <nm_private.h>
 
#ifdef PIOMAN
#include <pioman.h>
#endif /* PIOMAN */
 
#include "nm_mpi_ext.h"
#include "mpi.h"
 
#ifndef NM_MPI_H
#error including mpi.h does not include MadMPI; another mpi.h is in the way.
#endif /* NM_MPI_H */
 
#define nm_mpi_spinlock_t         nm_spinlock_t
#define nm_mpi_spin_init(LOCK)    nm_spin_init(LOCK)
#define nm_mpi_spin_destroy(LOCK) nm_spin_destroy(LOCK)
#define nm_mpi_spin_lock(LOCK)    nm_spin_lock(LOCK)
#define nm_mpi_spin_unlock(LOCK)  nm_spin_unlock(LOCK)
 
/* fixed-size floats */
#if SIZEOF_FLOAT==4
typedef float nm_mpi_float4_t;
#elif SIZEOF_DOUBLE==4
typedef double nm_mpi_float4_t;
#else
#  error "no C type for 4 bytes floats"
#endif
 
#if SIZEOF_FLOAT==8
typedef float nm_mpi_float8_t;
#elif SIZEOF_DOUBLE==8
typedef double nm_mpi_float8_t;
#else
#  error "no C type for 8 bytes floats"
#endif
 
typedef struct nm_mpi_errhandler_s
{
  int id;
  union
  {
    MPI_Handler_function*legacy;
    MPI_Comm_errhandler_function*comm_func;
    MPI_Win_errhandler_function*win_func;
    MPI_File_errhandler_function*file_func;
    MPI_Session_errhandler_function*session_func;
  } function;
  enum nm_mpi_errhandler_kind_e
    {
     NM_MPI_ERRHANDLER_NONE = 0,
     NM_MPI_ERRHANDLER_COMM,
     NM_MPI_ERRHANDLER_WIN,
     NM_MPI_ERRHANDLER_FILE,
     NM_MPI_ERRHANDLER_SESSION
    } kind;
} nm_mpi_errhandler_t;
 
extern struct nm_mpi_errhandler_s*nm_mpi_file_errhandler;
 
#define nm_mpi_errhandler_exec(P_ERRHANDLER, KIND, HANDLE, ERR)         \
  nm_mpi_errhandler_exec_internal(P_ERRHANDLER, KIND, HANDLE, ERR,      \
                                  __FUNCTION__, __FILE__, __LINE__)
 
#define NM_MPI_ERROR_INTERNAL(ERRORCODE) (ERRORCODE)
 
#define NM_MPI_ERROR_COMM(P_COMM, ERRORCODE)                            \
  ( ( ((ERRORCODE) == MPI_SUCCESS) ? MPI_SUCCESS :                      \
      nm_mpi_errhandler_exec((P_COMM)->p_errhandler, NM_MPI_ERRHANDLER_COMM, (P_COMM)->id, (ERRORCODE)) ) \
    , ERRORCODE )
 
#define NM_MPI_ERROR_SESSION(P_SESSION, ERRORCODE)                      \
  ( ( ((ERRORCODE) == MPI_SUCCESS) ? MPI_SUCCESS :                      \
      nm_mpi_errhandler_exec((P_SESSION)->p_errhandler, NM_MPI_ERRHANDLER_SESSION, (P_SESSION)->id, (ERRORCODE)) ) \
    , ERRORCODE )
 
#define NM_MPI_ERROR_WIN(P_WIN, ERRORCODE)                              \
  ( ( ((ERRORCODE) == MPI_SUCCESS) ? MPI_SUCCESS :                      \
      nm_mpi_errhandler_exec((P_WIN)->p_errhandler, NM_MPI_ERRHANDLER_WIN, (P_WIN)->id, (ERRORCODE)) ) \
    , ERRORCODE )
 
#define NM_MPI_ERROR_FILE(P_FILE, ERRORCODE)                            \
  ( ( ((ERRORCODE) == MPI_SUCCESS) ? MPI_SUCCESS :                      \
      nm_mpi_errhandler_exec((P_FILE)->p_errhandler, NM_MPI_ERRHANDLER_FILE, (P_FILE)->id, (ERRORCODE)) ) \
    , ERRORCODE )
 
#define NM_MPI_ERROR_FILE_DEFAULT(ERRORCODE)                            \
  ( ( ((ERRORCODE) == MPI_SUCCESS) ? MPI_SUCCESS :                      \
      nm_mpi_errhandler_exec(nm_mpi_file_errhandler, NM_MPI_ERRHANDLER_FILE, MPI_FILE_NULL, (ERRORCODE)) ) \
    , ERRORCODE )
 
#define NM_MPI_ERROR_WORLD(ERRORCODE)                                   \
  ( ( ((ERRORCODE) == MPI_SUCCESS) ? MPI_SUCCESS :                      \
      nm_mpi_errhandler_exec(nm_mpi_communicator_get(MPI_COMM_WORLD)->p_errhandler, NM_MPI_ERRHANDLER_COMM, MPI_COMM_WORLD, (ERRORCODE)) ) \
    , ERRORCODE )
 
#define NM_MPI_COMMUNICATOR_CHECK(P_COMM)                               \
  PUK_CHECK_TYPE(struct nm_mpi_communicator_s*, P_COMM);                \
  do { if((P_COMM) == NULL) return NM_MPI_ERROR_WORLD(MPI_ERR_COMM); } while(0)
 
#define NM_MPI_GROUP_CHECK(P_GROUP)                                     \
  PUK_CHECK_TYPE(struct nm_mpi_group_s*, P_GROUP);                      \
  do { if((P_GROUP) == NULL) return NM_MPI_ERROR_WORLD(MPI_ERR_GROUP); } while(0)
 
#define NM_MPI_SESSION_CHECK(P_SESSION)                                 \
  PUK_CHECK_TYPE(struct nm_mpi_session_s*, P_SESSION);                  \
  do { if((P_SESSION) == NULL) return NM_MPI_ERROR_WORLD(MPI_ERR_SESSION); } while(0)
 
#define NM_MPI_DATATYPE_CHECK(P_DATATYPE)                               \
  PUK_CHECK_TYPE(struct nm_mpi_datatype_s*, P_DATATYPE);                \
  do { if((P_DATATYPE) == NULL) return NM_MPI_ERROR_COMM(p_comm, MPI_ERR_TYPE); } while(0)
 
#define NM_MPI_TAG_CHECK_RECV(TAG)                                      \
  do { if(((TAG) != MPI_ANY_TAG) && ((TAG) < 0 || (TAG) > NM_MPI_TAG_MAX)) return NM_MPI_ERROR_COMM(p_comm, MPI_ERR_TAG); } while(0)
 
#define NM_MPI_TAG_CHECK_SEND(TAG)                                     \
  do { if( ((TAG) < 0) || ((TAG) > NM_MPI_TAG_MAX) || ((TAG) == MPI_ANY_TAG)) return NM_MPI_ERROR_COMM(p_comm, MPI_ERR_TAG); } while(0)
 
#define NM_MPI_WARNING(...) NM_WARN(__VA_ARGS__)
 
#define NM_MPI_FATAL_ERROR(...) NM_FATAL(__VA_ARGS__)
 
#define NM_MPI_TRACE(str, ...) NM_TRACEF(str, ## __VA_ARGS__)
 
#define FREE_AND_SET_NULL(p) do { padico_free(p); p = NULL; } while (0/*CONSTCOND*/)
 
static inline void nm_mpi_refcount_init(struct nm_refcount_s*p_refcount, const char*type __attribute__((unused)), int id)
{
  char*object_id = NULL;
#ifdef NMAD_DEBUG
  object_id = padico_string_fmt("type = %s; id = %d", type, id);
#endif
  nm_refcount_init(p_refcount, object_id, NULL);
}
 
 
#define NM_MPI_TAG_MAX                   ((nm_tag_t)(0x7FFFFFFF))
#define NM_MPI_TAG_PRIVATE_MASK          ((nm_tag_t)(0x80000000))
#define NM_MPI_TAG_PRIVATE_BASE          ((nm_tag_t)(0xF0000000))
#define NM_MPI_TAG_PRIVATE_BARRIER       ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x11))
#define NM_MPI_TAG_PRIVATE_BCAST         ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x12))
#define NM_MPI_TAG_PRIVATE_GATHER        ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x13))
#define NM_MPI_TAG_PRIVATE_GATHERV       ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x14))
#define NM_MPI_TAG_PRIVATE_SCATTER       ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x15))
#define NM_MPI_TAG_PRIVATE_SCATTERV      ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x16))
#define NM_MPI_TAG_PRIVATE_ALLTOALL      ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x17))
#define NM_MPI_TAG_PRIVATE_ALLTOALLV     ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x18))
#define NM_MPI_TAG_PRIVATE_REDUCE        ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x19))
#define NM_MPI_TAG_PRIVATE_ALLREDUCE     ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x1A))
#define NM_MPI_TAG_PRIVATE_SCAN          ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x1B))
#define NM_MPI_TAG_PRIVATE_REDUCESCATTER ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x1C))
#define NM_MPI_TAG_PRIVATE_ALLGATHER     ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x1D))
#define NM_MPI_TAG_PRIVATE_TYPE_ADD      ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x21)) 
#define NM_MPI_TAG_PRIVATE_TYPE_ADD_ACK  ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x22)) 
#define NM_MPI_TAG_PRIVATE_WIN_INIT      ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x23))
#define NM_MPI_TAG_PRIVATE_WIN_FENCE     ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x24))
#define NM_MPI_TAG_PRIVATE_WIN_BARRIER   ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x25))
#define NM_MPI_TAG_PRIVATE_COMM_SPLIT    ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0xF1))
#define NM_MPI_TAG_PRIVATE_COMM_CREATE   ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0xF2))
#define NM_MPI_TAG_PRIVATE_COMM_INFO     ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0xF3))
#define NM_MPI_TAG_PRIVATE_FILE_OPEN     ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0xF4))
#define NM_MPI_TAG_PRIVATE_RMA_BASE      ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x08000000))
#define NM_MPI_TAG_PRIVATE_RMA_BASE_SYNC ((nm_tag_t)(NM_MPI_TAG_PRIVATE_BASE | 0x04000000))
#define NM_MPI_TAG_PRIVATE_RMA_MASK_OP   ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE | 0x0F)) 
#define NM_MPI_TAG_PRIVATE_RMA_MASK_AOP  ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE | 0xF0)) 
#define NM_MPI_TAG_PRIVATE_RMA_MASK_FOP  ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_MASK_OP    | \
                                                     NM_MPI_TAG_PRIVATE_RMA_MASK_AOP))    
#define NM_MPI_TAG_PRIVATE_RMA_MASK_SEQ  ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE | 0xFFFF00))
#define NM_MPI_TAG_PRIVATE_RMA_MASK_WIN  ((nm_tag_t)(0xFFFFFFFF00000000 | NM_MPI_TAG_PRIVATE_RMA_BASE))
#define NM_MPI_TAG_PRIVATE_RMA_MASK_USER ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_MASK_FOP   | \
                                                     NM_MPI_TAG_PRIVATE_RMA_MASK_SEQ))
#define NM_MPI_TAG_PRIVATE_RMA_MASK      ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_MASK_WIN   | \
                                                     NM_MPI_TAG_PRIVATE_RMA_MASK_USER))
#define NM_MPI_TAG_PRIVATE_RMA_MASK_SYNC ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE_SYNC | 0xFFFFFFFF000000FF))
#define NM_MPI_TAG_PRIVATE_RMA_START     ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE_SYNC | 0x01))
#define NM_MPI_TAG_PRIVATE_RMA_END       ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE_SYNC | 0x02))
#define NM_MPI_TAG_PRIVATE_RMA_LOCK      ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE_SYNC | 0x13)) 
#define NM_MPI_TAG_PRIVATE_RMA_LOCK_R    ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE_SYNC | 0x04)) 
#define NM_MPI_TAG_PRIVATE_RMA_UNLOCK    ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE_SYNC | 0x05)) 
#define NM_MPI_TAG_PRIVATE_RMA_UNLOCK_R  ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE_SYNC | 0x06)) 
#define NM_MPI_TAG_PRIVATE_RMA_OP_CHECK  ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE_SYNC | 0x0F))
#define NM_MPI_TAG_PRIVATE_RMA_PUT       ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE | 0x1))
#define NM_MPI_TAG_PRIVATE_RMA_GET_REQ   ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE | 0x2))
#define NM_MPI_TAG_PRIVATE_RMA_ACC       ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE | 0x3)) 
#define NM_MPI_TAG_PRIVATE_RMA_GACC_REQ  ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE | 0x4)) 
#define NM_MPI_TAG_PRIVATE_RMA_FAO_REQ   ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE | 0x5)) 
#define NM_MPI_TAG_PRIVATE_RMA_CAS_REQ   ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE | 0x6)) 
#define NM_MPI_TAG_PRIVATE_RMA_REQ_RESP  ((nm_tag_t)(NM_MPI_TAG_PRIVATE_RMA_BASE | 0xF))
 
 
void nm_mpi_info_destructor(char*p_key, char*p_data);
 
PUK_HASHTABLE_TYPE(nm_mpi_info, char*, char*,
                   &puk_hash_string_default_hash, &puk_hash_string_default_eq, &nm_mpi_info_destructor);
 
struct nm_mpi_info_s
{
  int id; 
  struct nm_mpi_info_hashtable_s content; 
};
 
typedef void (nm_mpi_copy_subroutine_t)(int*id, int*keyval, void*extra_state,
                                        void*attribute_val_in, void*attribute_val_out, int*flag, int*ierr);
typedef void (nm_mpi_delete_subroutine_t)(int*id, int*keyval, void*attribute_val, void*extra_state, int*ierr);
 
typedef int (nm_mpi_attr_copy_fn_t)(int object_id, int keyval_id, void*extra_state,
                                    void*attribute_val_in, void*attribute_val_out, int*flag);
typedef int (nm_mpi_attr_delete_fn_t)(int object_id, int keyval_id, void*attribute_val, void*extra_state);
 
struct nm_mpi_keyval_s
{
  int id;                                         
  nm_mpi_attr_copy_fn_t*copy_fn;                  
  nm_mpi_attr_delete_fn_t*delete_fn;              
  nm_mpi_copy_subroutine_t*copy_subroutine;       
  nm_mpi_delete_subroutine_t*delete_subroutine;   
  void*extra_state;                               
  struct nm_refcount_s refcount;                  
};
 
void nm_mpi_attr_destructor(struct nm_mpi_keyval_s*p_keyval, void*p_value);
PUK_HASHTABLE_TYPE(nm_mpi_attr, struct nm_mpi_keyval_s*, void*,
                   &puk_hash_pointer_default_hash,
                   &puk_hash_pointer_default_eq,
                   &nm_mpi_attr_destructor);
 
struct nm_mpi_attrs_s
{
  struct nm_mpi_attr_hashtable_s table;
  int id; 
};
 
typedef struct nm_mpi_group_s
{
  int id;                 
  nm_group_t p_nm_group;  
} nm_mpi_group_t;
 
typedef struct nm_mpi_communicator_s
{
  int id;                            
  nm_comm_t p_nm_comm;               
  struct nm_mpi_attrs_s attrs;       
  struct nm_mpi_info_s*p_info;       
  nm_mpi_errhandler_t*p_errhandler;  
  char*name;                         
  struct nm_refcount_s refcount;     
  enum nm_mpi_communicator_kind_e
    {
      NM_MPI_COMMUNICATOR_UNSPEC = 0,
      NM_MPI_COMMUNICATOR_INTRA  = 1,
      NM_MPI_COMMUNICATOR_INTER
    } kind;
  enum nm_mpi_topology_kind_e
    {
      NM_MPI_TOPOLOGY_NONE  = MPI_UNDEFINED,
      NM_MPI_TOPOLOGY_GRAPH = MPI_GRAPH,
      NM_MPI_TOPOLOGY_CART  = MPI_CART
    } topo;
  struct nm_mpi_cart_topology_s  
  {
    int ndims;    
    int*dims;     
    int*periods;  
    int size;     
  } cart_topology;
  struct nm_mpi_intercomm_s
  {
    nm_group_t p_local_group;
    nm_group_t p_remote_group;
    int rank;          
    int remote_offset; 
  } intercomm;
} nm_mpi_communicator_t;
struct nm_mpi_session_s
{
  int id;
  nm_mpi_errhandler_t*p_errhandler;  
  struct nm_mpi_info_s*p_info;
};
struct nm_mpi_window_s;
typedef struct nm_mpi_window_s nm_mpi_window_t;
struct nm_mpi_win_epoch_s;
typedef struct nm_mpi_win_epoch_s nm_mpi_win_epoch_t;
typedef int32_t nm_mpi_request_type_t;
#define NM_MPI_REQUEST_ZERO       ((nm_mpi_request_type_t)0x0000)
#define NM_MPI_REQUEST_RECV       ((nm_mpi_request_type_t)0x0001)
#define NM_MPI_REQUEST_PRECV      ((nm_mpi_request_type_t)0x0041)
#define NM_MPI_REQUEST_PROBE      ((nm_mpi_request_type_t)0x0002)
#define NM_MPI_REQUEST_SEND       ((nm_mpi_request_type_t)0x0004)
#define NM_MPI_REQUEST_RSEND      ((nm_mpi_request_type_t)0x0014)
#define NM_MPI_REQUEST_SSEND      ((nm_mpi_request_type_t)0x0024)
#define NM_MPI_REQUEST_PSEND      ((nm_mpi_request_type_t)0x0044)
#define NM_MPI_REQUEST_FILE       ((nm_mpi_request_type_t)0x0008)
#define NM_MPI_REQUEST_GREQUEST   ((nm_mpi_request_type_t)0x0080)
#define NM_MPI_REQUEST_IBARRIER   ((nm_mpi_request_type_t)0x1100)
#define NM_MPI_REQUEST_IBCAST     ((nm_mpi_request_type_t)0x2100)
#define NM_MPI_REQUEST_IREDUCE    ((nm_mpi_request_type_t)0x3100)
#define NM_MPI_REQUEST_IALLREDUCE ((nm_mpi_request_type_t)0x4100)
#define NM_MPI_REQUEST_IALLTOALL  ((nm_mpi_request_type_t)0x5100)
#define NM_MPI_REQUEST_IALLTOALLV ((nm_mpi_request_type_t)0x6100)
#define NM_MPI_REQUEST_IGATHER    ((nm_mpi_request_type_t)0x7100)
#define NM_MPI_REQUEST_IALLGATHER ((nm_mpi_request_type_t)0x8100)
 
#define NM_MPI_REQUEST_COLLECTIVE  ((nm_mpi_request_type_t)0x0100)
#define NM_MPI_REQUEST_PARTITIONED ((nm_mpi_request_type_t)0x0040)
 
typedef int8_t nm_mpi_status_t;
#define NM_MPI_STATUS_NONE        ((nm_mpi_status_t)0x00)
#define NM_MPI_REQUEST_CANCELLED  ((nm_mpi_status_t)0x04) 
#define NM_MPI_REQUEST_PERSISTENT ((nm_mpi_status_t)0x08) 
#define _NM_MPI_MAX_DATATYPE_SIZE 64
 
PUK_LIST_DECLARE_TYPE(nm_mpi_request);
 
typedef struct nm_mpi_request_s
{
  MPI_Request id;
  nm_sr_request_t request_nmad;
  nm_mpi_request_type_t request_type;
  nm_mpi_status_t status;
  int user_tag;
  int request_source;
  int request_error;
  struct nm_mpi_datatype_s*p_datatype;
  struct nm_mpi_datatype_s*p_datatype2;
  nm_gate_t gate;
  nm_mpi_communicator_t*p_comm;
  nm_mpi_count_t count;
  union
  {
    void*rbuf;       
    const void*sbuf; 
    char static_buf[_NM_MPI_MAX_DATATYPE_SIZE]; 
    struct nm_mpi_file_op_s*p_file_op;          
    struct nm_mpi_grequest_s*p_grequest;        
  };
  union
  {
    struct
    {
      struct nm_coll_req_s*p_coll_req; 
      union
      {
        struct
        {
          int root;
        } bcast;
        struct
        {
          int root;
          const void*sendbuf;
          void*recvbuf;
          struct nm_mpi_operator_s*p_op;
        } reduce;
        struct
        {
          const void*sendbuf;
          void*recvbuf;
          struct nm_mpi_operator_s*p_op;
        } allreduce;
        struct
        {
          int root;
          struct nm_data_s send_data;
          struct nm_data_s*p_recv_data;
        } gather;
        struct
        {
          const void*sendbuf;
          void*recvbuf;
          nm_mpi_count_t sendcount, recvcount;
          struct nm_mpi_datatype_s*p_sendtype, *p_recvtype;
        } allgather;
        struct
        {
          const void*sendbuf;
          void*recvbuf;
          nm_mpi_count_t sendcount, recvcount;
          struct nm_mpi_datatype_s*p_sendtype, *p_recvtype;
        } alltoall;
        struct
        {
          const void*sendbuf;
          nm_mpi_count_t*p_sendcounts;
          nm_mpi_aint_t*p_sdispls;
          struct nm_mpi_datatype_s*p_sendtype;
          void*recvbuf;
          nm_mpi_count_t*p_recvcounts;
          nm_mpi_aint_t*p_rdispls;
          struct nm_mpi_datatype_s*p_recvtype;
        } alltoallv;
      };
    } collective;
    struct
    {
      int partitions;             
    } partitioned;
    struct nm_mpi_grequest_s
    {
      MPI_Grequest_query_function*query_fn;
      MPI_Grequest_free_function*free_fn;
      MPI_Grequest_cancel_function*cancel_fn;
      void*extra_state;
      nm_cond_status_t completed; 
    } grequest;
    struct
    {
      nm_mpi_win_epoch_t*p_epoch; 
      nm_mpi_window_t*p_win;      
    } rma; 
  };
  PUK_LIST_LINK(nm_mpi_request);
} __attribute__((__may_alias__)) nm_mpi_request_t;
 
PUK_LIST_CREATE_FUNCS(nm_mpi_request);
 
typedef struct nm_mpi_operator_s
{
  int id;
  MPI_User_function*function;
  MPI_User_function_c*function_c;
  int commute;
} nm_mpi_operator_t;
typedef uint32_t nm_mpi_datatype_hash_t;
 
typedef struct nm_mpi_datatype_ser_s
{
  nm_mpi_type_combiner_t combiner;
  nm_mpi_count_t count;
  size_t ser_size;
  nm_mpi_datatype_hash_t hash;
  union nm_mpi_datatype_ser_param_u
  {
    struct nm_mpi_datatype_ser_param_dup_s
    {
      nm_mpi_datatype_hash_t old_type;
    } DUP;
    struct nm_mpi_datatype_ser_param_resized_s
    {
      nm_mpi_datatype_hash_t old_type;
      nm_mpi_count_t lb;
      nm_mpi_count_t extent;
    } RESIZED;
    struct nm_mpi_datatype_ser_param_contiguous_s
    {
      nm_mpi_datatype_hash_t old_type;
    } CONTIGUOUS;
    struct nm_mpi_datatype_ser_param_vector_s
    {
      nm_mpi_datatype_hash_t old_type;
      nm_mpi_count_t stride;       
      nm_mpi_count_t blocklength;
    } VECTOR;
    struct nm_mpi_datatype_ser_param_hvector_s
    {
      nm_mpi_datatype_hash_t old_type;
      nm_mpi_count_t hstride; 
      nm_mpi_count_t blocklength;
    } HVECTOR;
    struct nm_mpi_datatype_ser_param_indexed_s
    {
      nm_mpi_datatype_hash_t old_type;
      void*DATA;
    } INDEXED;
    struct nm_mpi_datatype_ser_param_hindexed_s
    {
      nm_mpi_datatype_hash_t old_type;
      void*DATA;
    } HINDEXED;
    struct nm_mpi_datatype_ser_param_indexed_block_s
    {
      nm_mpi_datatype_hash_t old_type;
      int blocklength;
      void*DATA;
    } INDEXED_BLOCK;
    struct nm_mpi_datatype_ser_param_hindexed_block_s
    {
      nm_mpi_datatype_hash_t old_type;
      int blocklength;
      void*DATA;
    } HINDEXED_BLOCK;
    struct nm_mpi_datatype_ser_param_subarray_s
    {
      nm_mpi_datatype_hash_t old_type;
      int ndims;
      int order;
      void*DATA;
    } SUBARRAY;
    struct nm_mpi_datatype_ser_param_struct_s
    {
      void*DATA;
    } STRUCT;
  } p;
} nm_mpi_datatype_ser_t;
 
typedef struct nm_mpi_datatype_s
{
  int id;
  nm_mpi_type_combiner_t combiner;
  nm_mpi_count_t count;
  int is_compact;
  int is_contig;
  nm_mpi_count_t elements;
  nm_mpi_count_t lb;
  nm_mpi_count_t extent;
  nm_mpi_count_t true_lb;
  nm_mpi_count_t true_extent;
  size_t size;
  struct nm_data_properties_s props;
  struct nm_refcount_s refcount;
  int committed;
  union
  {
    struct
    {
      struct nm_mpi_datatype_s*p_old_type;
    } DUP;
    struct
    {
      struct nm_mpi_datatype_s*p_old_type;
    } RESIZED;
    struct
    {
      struct nm_mpi_datatype_s*p_old_type;
    } CONTIGUOUS;
    struct
    {
      struct nm_mpi_datatype_s*p_old_type;
      nm_mpi_count_t blocklength;
      nm_mpi_aint_t stride;       
    } VECTOR;
    struct
    {
      struct nm_mpi_datatype_s*p_old_type;
      nm_mpi_count_t blocklength;
      nm_mpi_aint_t hstride; 
    } HVECTOR;
    struct
    {
      struct nm_mpi_datatype_s*p_old_type;
      struct nm_mpi_type_indexed_map_s
      {
        nm_mpi_count_t blocklength;
        nm_mpi_aint_t displacement; 
      } *p_map;
    } INDEXED;
    struct
    {
      struct nm_mpi_datatype_s*p_old_type;
      struct nm_mpi_type_hindexed_map_s
      {
        nm_mpi_count_t blocklength;
        nm_mpi_aint_t displacement; 
      } *p_map;
    } HINDEXED;
    struct
    {
      struct nm_mpi_datatype_s*p_old_type;
      nm_mpi_count_t blocklength;
      nm_mpi_count_t*array_of_displacements;
    } INDEXED_BLOCK;
    struct
    {
      struct nm_mpi_datatype_s*p_old_type;
      nm_mpi_count_t blocklength;
      nm_mpi_aint_t*array_of_displacements;
    } HINDEXED_BLOCK;
    struct
    {
      struct nm_mpi_datatype_s*p_old_type;
      int ndims;
      int order;
      struct nm_mpi_type_subarray_dim_s
      {
        int size;
        int subsize;
        int start;
      } *p_dims;
    } SUBARRAY;
    struct
    {
      struct nm_mpi_type_struct_map_s
      {
        struct nm_mpi_datatype_s*p_old_type;
        nm_mpi_count_t blocklength;
        nm_mpi_aint_t displacement; 
      } *p_map;
    } STRUCT;
  };
  char*name;
  struct nm_mpi_attrs_s attrs; 
  size_t ser_size;       
  nm_mpi_datatype_ser_t*p_serialized; 
  nm_mpi_datatype_hash_t hash;        
} nm_mpi_datatype_t;
 
struct nm_data_mpi_datatype_s
{
  void*ptr;                             
  struct nm_mpi_datatype_s*p_datatype;  
  nm_mpi_count_t count;                 
};
__PUK_SYM_INTERNAL extern const struct nm_data_ops_s nm_mpi_datatype_ops;
NM_DATA_TYPE(mpi_datatype, struct nm_data_mpi_datatype_s, &nm_mpi_datatype_ops);
 
typedef void (*nm_mpi_datatype_apply_t)(void*ptr, nm_len_t len, void*_ref);
 
struct nm_data_mpi_datatype_header_s
{
  nm_mpi_aint_t offset;        
  nm_mpi_datatype_hash_t datatype_hash; 
  nm_mpi_count_t count;   
};
 
struct nm_data_mpi_datatype_wrapper_s
{
  struct nm_data_mpi_datatype_header_s header; 
  struct nm_data_mpi_datatype_s data;          
};
__PUK_SYM_INTERNAL extern const struct nm_data_ops_s nm_mpi_datatype_wrapper_ops;
NM_DATA_TYPE(mpi_datatype_wrapper, struct nm_data_mpi_datatype_wrapper_s, &nm_mpi_datatype_wrapper_ops);
 
__PUK_SYM_INTERNAL extern const struct nm_data_ops_s nm_mpi_datatype_serialize_ops;
NM_DATA_TYPE(mpi_datatype_serial, struct nm_mpi_datatype_s*, &nm_mpi_datatype_serialize_ops);
 
typedef struct nm_mpi_win_locklist_s
{
  nm_mpi_spinlock_t lock;       
  struct nm_mpi_request_list_s pending; 
} nm_mpi_win_locklist_t;
 
struct nm_mpi_win_epoch_s
{
  volatile int mode;  
  uint64_t nmsg;      
  uint64_t completed; 
};
 
typedef struct nm_mpi_win_pass_mngmt_s
{
  uint64_t excl_pending;   
  uint16_t lock_type;      
  uint64_t naccess;        
  nm_mpi_win_locklist_t q; 
} nm_mpi_win_pass_mngmt_t;
 
struct nm_mpi_window_s
{
  int id;
  int myrank;
  void*p_base;
  nm_mpi_aint_t*size;
  int*disp_unit;
  int*win_ids;
  nm_mpi_communicator_t*p_comm;
  nm_group_t p_group;
  struct nm_sr_monitor_s monitor;
  struct nm_sr_monitor_s monitor_sync;
  struct nm_mpi_info_s*p_info;
  int flavor;
  int mode;
  int mem_model;
  nm_mpi_errhandler_t*p_errhandler;
  char*name;
  struct nm_mpi_attrs_s attrs;
  uint16_t next_seq;
  nm_mpi_win_epoch_t*access;
  nm_mpi_win_epoch_t*exposure;
  nm_mpi_request_t**end_reqs;
  uint64_t*msg_count;
  nm_mpi_win_pass_mngmt_t waiting_queue;
  nm_mpi_win_locklist_t*pending_ops;
  nm_mpi_spinlock_t lock;
  char shared_file_name[32];
};
 
#define NM_MPI_WIN_UNUSED                 0
#define NM_MPI_WIN_ACTIVE_TARGET          1
#define NM_MPI_WIN_PASSIVE_TARGET         2
#define NM_MPI_WIN_PASSIVE_TARGET_END     4
#define NM_MPI_WIN_PASSIVE_TARGET_PENDING 8
#define NM_MPI_LOCK_NONE      ((uint16_t)0)
#define NM_MPI_LOCK_EXCLUSIVE ((uint16_t)MPI_LOCK_EXCLUSIVE)
#define NM_MPI_LOCK_SHARED    ((uint16_t)MPI_LOCK_SHARED)
#define _NM_MPI_LOCK_OFFSET   ((uint16_t)3)
#ifdef NMAD_PROFILE
struct nm_mpi_profiling_s
{
  unsigned long cur_req_send, max_req_send;
  unsigned long cur_req_recv, max_req_recv;
  unsigned long cur_req_total, max_req_total;
};
extern struct nm_mpi_profiling_s nm_mpi_profile;
#endif /* NMAD_PROFILE */
 
/* ********************************************************* */
 
#ifdef NMAD_DEBUG
#define NM_MPI_HANDLE_DEBUG 1
#else /* NMAD_DEBUG */
#define NM_MPI_HANDLE_DEBUG 0
#endif /* NMAD_DEBUG */
 
 
#define NM_MPI_HANDLE_TYPE(ENAME, TYPE, OFFSET, SLABSIZE)               \
  PUK_VECT_TYPE(nm_mpi_entry_ ## ENAME, TYPE*);                         \
  NM_ALLOCATOR_TYPE(ENAME, TYPE);                                       \
  struct nm_mpi_handle_##ENAME##_s                                      \
  {                                                                     \
    struct nm_mpi_entry_##ENAME##_vect_s table;                         \
    struct puk_int_vect_s pool;   \
    int next_id;                 \
    nm_mpi_spinlock_t lock;                                             \
    struct ENAME##_allocator_s allocator;                               \
    char*(*display)(TYPE*);                                             \
  };                                                                    \
  __attribute__((unused))                                               \
  static int nm_mpi_handle_##ENAME##_initialized(struct nm_mpi_handle_##ENAME##_s*p_allocator) \
  {                                                                     \
    return (p_allocator->next_id != -1);                                \
  }                                                                     \
                         \
  __attribute__((unused))                                               \
  static int nm_mpi_handle_##ENAME##_load(struct nm_mpi_handle_##ENAME##_s*p_allocator) \
  {                                                                     \
    int load = 0;                                                       \
    nm_mpi_entry_##ENAME##_vect_itor_t i;                               \
    puk_vect_foreach(i, nm_mpi_entry_##ENAME, &p_allocator->table)      \
      {                                                                 \
        if(*i != NULL)                                                  \
          {                                                             \
            load++;                                                     \
          }                                                             \
      }                                                                 \
    return load;                                                        \
    }                                                                   \
        \
  __attribute__((unused))                                               \
  static void nm_mpi_handle_##ENAME##_dump(struct nm_mpi_handle_##ENAME##_s*p_allocator) \
  {                                                                     \
    NM_MPI_TRACE("dump pending objects of kind %s\n", #ENAME);          \
    if( nm_mpi_handle_##ENAME##_initialized(p_allocator) &&             \
        !nm_mpi_entry_##ENAME##_vect_empty(&p_allocator->table))        \
      {                                                                 \
        nm_mpi_entry_##ENAME##_vect_itor_t i;                           \
        puk_vect_foreach(i, nm_mpi_entry_##ENAME, &p_allocator->table)  \
          {                                                             \
            if(*i != NULL)                                              \
              {                                                         \
                const int index = nm_mpi_entry_##ENAME##_vect_rank(&p_allocator->table, i); \
                if(index != (*i)->id)                                   \
                  {                                                     \
                    NM_MPI_WARNING("inconsistency in " #ENAME " entry; index = %d; id = %d\n", index, (*i)->id); \
                  }                                                     \
                if(index >= (OFFSET))                                   \
                  {                                                     \
                    char*d = NULL;                                      \
                    if(p_allocator->display != NULL)                    \
                      {                                                 \
                        d = (*p_allocator->display)(*i);                \
                      }                                                 \
                    NM_MPI_WARNING("pending object; type = " #ENAME "; id = %d; ptr = %p; %s\n", (*i)->id, *i, \
                                   (d ? d : ""));                       \
                    if(d) padico_free(d);                               \
                  }                                                     \
              }                                                         \
          }                                                             \
      }                                                                 \
  }                                                                     \
                                       \
  __attribute__((unused))                                               \
  static void nm_mpi_handle_##ENAME##_init(struct nm_mpi_handle_##ENAME##_s*p_allocator) \
  {                                                                     \
    nm_mpi_entry_##ENAME##_vect_init(&p_allocator->table);              \
    puk_int_vect_init(&p_allocator->pool);                              \
    p_allocator->next_id = OFFSET;                                      \
    nm_mpi_spin_init(&p_allocator->lock);                               \
    p_allocator->display = NULL;                                        \
    ENAME##_allocator_init(&p_allocator->allocator, SLABSIZE);          \
    int i;                                                              \
    for(i = 0; i < OFFSET; i++)                                         \
      {                                                                 \
        nm_mpi_entry_##ENAME##_vect_push_back(&p_allocator->table, NULL); \
      }                                                                 \
    nm_mpi_cleanup_register((void (*)(void*))&nm_mpi_handle_ ## ENAME ## _dump, p_allocator); \
  }                                                                     \
  __attribute__((unused))                                               \
  static void nm_mpi_handle_##ENAME##_finalize(struct nm_mpi_handle_##ENAME##_s*p_allocator, void(*destructor)(TYPE*)) \
  {                                                                     \
    nm_mpi_cleanup_unregister((void (*)(void*))&nm_mpi_handle_##ENAME##_dump, p_allocator); \
    if(NM_MPI_HANDLE_DEBUG)                                             \
      nm_mpi_handle_##ENAME##_dump(p_allocator);                        \
    if(destructor != NULL)                                              \
      {                                                                 \
        nm_mpi_entry_##ENAME##_vect_itor_t i;                           \
        puk_vect_foreach(i, nm_mpi_entry_##ENAME, &p_allocator->table)  \
          {                                                             \
            if(*i != NULL)                                              \
              {                                                         \
                (*destructor)(*i);                                      \
              }                                                         \
          }                                                             \
      }                                                                 \
    nm_mpi_entry_##ENAME##_vect_destroy(&p_allocator->table);           \
    puk_int_vect_destroy(&p_allocator->pool);                           \
    ENAME##_allocator_destroy(&p_allocator->allocator);                 \
  }                                                                     \
                           \
  __attribute__((unused))                                               \
  static TYPE*nm_mpi_handle_##ENAME##_store(struct nm_mpi_handle_##ENAME##_s*p_allocator, int id) \
  {                                                                     \
    assert(nm_mpi_handle_##ENAME##_initialized(p_allocator));           \
    TYPE*e = ENAME##_malloc(&p_allocator->allocator);                   \
    if((id <= 0) || (id > OFFSET))                                      \
      {                                                                 \
        NM_MPI_FATAL_ERROR("madmpi: cannot store invalid %s handle id %d\n", #ENAME, id); \
      }                                                                 \
    if(nm_mpi_entry_##ENAME##_vect_at(&p_allocator->table, id) != NULL) \
      {                                                                 \
        NM_MPI_FATAL_ERROR("madmpi: %s handle %d busy; cannot store.", #ENAME, id); \
      }                                                                 \
    nm_mpi_entry_##ENAME##_vect_put(&p_allocator->table, e, id);        \
    e->id = id;                                                         \
    return e;                                                           \
  }                                                                     \
                                       \
  __attribute__((unused))                                               \
  static TYPE*nm_mpi_handle_##ENAME##_alloc(struct nm_mpi_handle_##ENAME##_s*p_allocator) \
  {                                                                     \
    assert(nm_mpi_handle_##ENAME##_initialized(p_allocator));           \
    TYPE*e = ENAME##_malloc(&p_allocator->allocator);                   \
    int new_id = -1;                                                    \
    nm_mpi_spin_lock(&p_allocator->lock);                               \
    if(puk_int_vect_empty(&p_allocator->pool))                          \
      {                                                                 \
        new_id = p_allocator->next_id++;                                \
        nm_mpi_entry_##ENAME##_vect_resize(&p_allocator->table, new_id); \
      }                                                                 \
    else                                                                \
      {                                                                 \
        new_id = puk_int_vect_pop_back(&p_allocator->pool);             \
      }                                                                 \
    nm_mpi_entry_##ENAME##_vect_put(&p_allocator->table, e, new_id);    \
    nm_mpi_spin_unlock(&p_allocator->lock);                             \
    e->id = new_id;                                                     \
    return e;                                                           \
  }                                                                     \
                          \
  __attribute__((unused))                                               \
  static void nm_mpi_handle_##ENAME##_release(struct nm_mpi_handle_##ENAME##_s*p_allocator, TYPE*e) \
  {                                                                     \
    const int id = e->id;                                               \
    assert(id > 0);                                                     \
    nm_mpi_spin_lock(&p_allocator->lock);                               \
    assert(nm_mpi_entry_##ENAME##_vect_at(&p_allocator->table, id) == e); \
    nm_mpi_entry_##ENAME##_vect_put(&p_allocator->table, NULL, id);     \
    puk_int_vect_push_back(&p_allocator->pool, id);                     \
    e->id = -1;                                                         \
    nm_mpi_spin_unlock(&p_allocator->lock);                             \
  }                                                                     \
                                                 \
  __attribute__((unused))                                               \
  static void nm_mpi_handle_##ENAME##_free(struct nm_mpi_handle_##ENAME##_s*p_allocator, TYPE*e) \
  {                                                                     \
    if(e->id > 0)                                                       \
      nm_mpi_handle_##ENAME##_release(p_allocator, e);                  \
    ENAME##_free(&p_allocator->allocator, e);                           \
  }                                                                     \
                          \
  __attribute__((unused))                                               \
  static TYPE*nm_mpi_handle_##ENAME##_get(struct nm_mpi_handle_##ENAME##_s*p_allocator, int id) \
  {                                                                     \
    assert(nm_mpi_handle_##ENAME##_initialized(p_allocator));           \
    TYPE*e = NULL;                                                      \
    if((id > 0) && (id < nm_mpi_entry_##ENAME##_vect_size(&p_allocator->table))) \
      {                                                                 \
        nm_mpi_spin_lock(&p_allocator->lock);                           \
        e = nm_mpi_entry_##ENAME##_vect_at(&p_allocator->table, id);    \
        nm_mpi_spin_unlock(&p_allocator->lock);                         \
      }                                                                 \
    else                                                                \
      {                                                                 \
        return NULL;                                                    \
      }                                                                 \
    return e;                                                           \
  }                                                                     \
  __attribute__((unused))                                               \
  static void nm_mpi_handle_##ENAME##_set_display(struct nm_mpi_handle_##ENAME##_s*p_allocator, char*(*display)(TYPE*)) \
  {                                                                     \
    p_allocator->display = display;                                     \
  }
 
#define NM_MPI_HANDLE_NULL { .next_id = -1 }
 
 
/* ********************************************************* */
 
/* ** submodules init/exit */
 
void nm_mpi_err_lazy_init(void);
void nm_mpi_err_lazy_exit(void);
 
void nm_mpi_request_lazy_init(void);
 
void nm_mpi_request_lazy_exit(void);
 
void nm_mpi_comm_lazy_init(void);
 
void nm_mpi_comm_lazy_exit(void);
 
void nm_mpi_session_lazy_init(void);
 
void nm_mpi_session_lazy_exit(void);
 
void nm_mpi_group_lazy_init(void);
 
void nm_mpi_group_lazy_exit(void);
 
void nm_mpi_datatype_lazy_init(void);
void nm_mpi_datatype_lazy_exit(void);
 
void nm_mpi_datatype_exchange_init(void);
void nm_mpi_datatype_exchange_exit(void);
 
void nm_mpi_op_lazy_init(void);
 
void nm_mpi_op_lazy_exit(void);
 
void nm_mpi_io_init(void);
 
void nm_mpi_io_exit(void);
 
void nm_mpi_attrs_lazy_init(void);
 
void nm_mpi_attrs_lazy_exit(void);
 
void nm_mpi_win_init(void);
 
void nm_mpi_win_exit(void);
 
void nm_mpi_rma_init(void);
 
void nm_mpi_rma_exit(void);
 
void nm_mpi_info_lazy_init(void);
void nm_mpi_info_lazy_exit(void);
 
void nm_mpi_cleanup_register(void(*p_func)(void*), void*p_arg);
void nm_mpi_cleanup_unregister(void(*p_func)(void*), void*p_arg);
 
 
/* ** Communicators **************************************** */
 
static inline nm_gate_t nm_mpi_communicator_get_gate(nm_mpi_communicator_t*p_comm, int node)
{
  assert(p_comm->kind != NM_MPI_COMMUNICATOR_UNSPEC);
  if(p_comm->kind == NM_MPI_COMMUNICATOR_INTRA)
    return nm_comm_get_gate(p_comm->p_nm_comm, node);
  else if(p_comm->kind == NM_MPI_COMMUNICATOR_INTER)
    return nm_group_get_gate(p_comm->intercomm.p_remote_group, node);
  else
    NM_MPI_FATAL_ERROR("wrong kind %d for communicator %d.\n", p_comm->kind, p_comm->id);
  return NULL;
}
 
static inline int nm_mpi_communicator_get_dest(nm_mpi_communicator_t*p_comm, nm_gate_t p_gate)
{
  int rank = nm_comm_get_dest(p_comm->p_nm_comm, p_gate);
  if(p_comm->kind == NM_MPI_COMMUNICATOR_INTER)
    {
      rank -= p_comm->intercomm.remote_offset;
      assert(rank < nm_group_size(p_comm->intercomm.p_remote_group));
      assert(rank >= 0);
    }
  return rank;
}
 
static inline nm_session_t nm_mpi_communicator_get_session(nm_mpi_communicator_t*p_comm)
{
  return nm_comm_get_session(p_comm->p_nm_comm);
}
 
 
/* ** Groups *********************************************** */
 
nm_mpi_group_t*nm_mpi_group_alloc(void);
 
nm_mpi_group_t*nm_mpi_group_get(MPI_Group group);
 
 
/* ** Err handlers ***************************************** */
 
struct nm_mpi_errhandler_s*nm_mpi_errhandler_alloc(enum nm_mpi_errhandler_kind_e kind, MPI_Handler_function*func);
 
struct nm_mpi_errhandler_s*nm_mpi_errhandler_get(int errhandler, enum nm_mpi_errhandler_kind_e kind);
 
void nm_mpi_errhandler_exec_internal(struct nm_mpi_errhandler_s*p_errhandler, enum nm_mpi_errhandler_kind_e kind, int handle, int err,
                                     const char*func, const char*file, const int line);
 
int nm_mpi_err_lastused(void);
 
/* ** Info Operations ************************************** */
 
struct nm_mpi_info_s*nm_mpi_info_alloc(void);
 
void nm_mpi_info_free(struct nm_mpi_info_s*p_info);
 
struct nm_mpi_info_s*nm_mpi_info_get(MPI_Info info);
 
void nm_mpi_info_define(struct nm_mpi_info_s*p_info, const char*p_key, const char*p_value);
 
struct nm_mpi_info_s*nm_mpi_info_dup(struct nm_mpi_info_s*p_info);
 
void nm_mpi_info_copy(struct nm_mpi_info_s*p_dest_info, struct nm_mpi_info_s*p_src_info);
 
void nm_mpi_info_update(struct nm_mpi_info_s*p_info_up, struct nm_mpi_info_s*p_info_origin);
 
 
/* ** Datatype functionalities ***************************** */
 
size_t nm_mpi_datatype_size(nm_mpi_datatype_t*p_datatype);
 
 
nm_mpi_count_t nm_mpi_datatype_get_elements(nm_mpi_datatype_t*p_datatype, nm_len_t size);
 
nm_mpi_datatype_t*nm_mpi_datatype_get(MPI_Datatype datatype);
 
void nm_mpi_datatype_properties_compute(nm_mpi_datatype_t*p_datatype);
 
void nm_mpi_datatype_traversal_apply(const void*_content, struct nm_data_op_s*p_op);
 
nm_mpi_datatype_t*nm_mpi_datatype_hashtable_get(nm_mpi_datatype_hash_t datatype_hash);
 
#define nm_mpi_datatype_ref_inc(P_DATATYPE, P_HOLDER) nm_refcount_inc(&(P_DATATYPE)->refcount, (P_HOLDER))
 
 
int nm_mpi_datatype_ref_dec(nm_mpi_datatype_t*p_datatype, const void*p_holder);
 
static inline void*nm_mpi_datatype_get_ptr(void*buf, nm_mpi_count_t count, const nm_mpi_datatype_t*p_datatype)
{
  return buf + count * p_datatype->extent;
}
 
static inline void nm_mpi_data_build(struct nm_data_s*p_data, void*ptr, struct nm_mpi_datatype_s*p_datatype, nm_mpi_count_t count)
{
  nm_data_mpi_datatype_set(p_data, (struct nm_data_mpi_datatype_s){ .ptr = ptr, .p_datatype = p_datatype, .count = count });
}
 
static inline void nm_mpi_datatype_pack(void*outbuf, const void*inbuf, nm_mpi_datatype_t*p_datatype, nm_mpi_count_t count)
{
  struct nm_data_s data;
  nm_mpi_data_build(&data, (void*)inbuf, p_datatype, count);
  nm_data_copy_from(&data, 0 /* offset */, nm_data_size(&data), outbuf);
}
 
static inline void nm_mpi_datatype_unpack(const void*inbuf, void*outbuf, nm_mpi_datatype_t*p_datatype, nm_mpi_count_t count)
{
  struct nm_data_s data;
  nm_mpi_data_build(&data, outbuf, p_datatype, count);
  nm_data_copy_to(&data, 0 /* offset */, nm_data_size(&data), inbuf);
}
 
static inline void nm_mpi_datatype_copy(const void*src_buf, nm_mpi_datatype_t*p_src_type, nm_mpi_count_t src_count,
                                        void*dest_buf, nm_mpi_datatype_t*p_dest_type, nm_mpi_count_t dest_count)
{
  struct nm_data_s src_data;
  nm_mpi_data_build(&src_data, (void*)src_buf, p_src_type, src_count);
  struct nm_data_s dest_data;
  nm_mpi_data_build(&dest_data, dest_buf, p_dest_type, dest_count);
  nm_data_copy(&dest_data, &src_data);
}
 
void nm_mpi_datatype_deserialize(nm_mpi_datatype_ser_t*p_datatype, MPI_Datatype*p_newtype);
 
int nm_mpi_datatype_send(nm_gate_t gate, nm_mpi_datatype_t*p_datatype);
 
void nm_mpi_datatype_hashtable_insert(nm_mpi_datatype_t*p_datatype);
 
static inline nm_mpi_datatype_hash_t nm_mpi_datatype_hash_common(const nm_mpi_datatype_t*p_datatype)
{
  nm_mpi_datatype_hash_t z = puk_hash_oneatatime((const void*)&p_datatype->combiner, sizeof(nm_mpi_type_combiner_t))
    + puk_hash_oneatatime((const void*)&p_datatype->count, sizeof(MPI_Count))
    + puk_hash_oneatatime((const void*)&p_datatype->lb, sizeof(MPI_Aint))
    + puk_hash_oneatatime((const void*)&p_datatype->extent, sizeof(MPI_Aint))
    + puk_hash_oneatatime((const void*)&p_datatype->size, sizeof(size_t));
  return z;
}
 
/* ** Requests functions *********************************** */
 
nm_mpi_request_t*nm_mpi_request_alloc(void);
 
nm_mpi_request_t*nm_mpi_request_alloc_send(nm_mpi_request_type_t type, nm_mpi_count_t count, const void*sbuf,
                                           struct nm_mpi_datatype_s*p_datatype, int tag, struct nm_mpi_communicator_s*p_comm);
 
nm_mpi_request_t*nm_mpi_request_alloc_recv(nm_mpi_count_t count, void*rbuf,
                                           struct nm_mpi_datatype_s*p_datatype, int tag, struct nm_mpi_communicator_s*p_comm);
 
nm_mpi_request_t*nm_mpi_request_alloc_icol(nm_mpi_request_type_t type, nm_mpi_count_t count, struct nm_mpi_datatype_s*p_datatype,
                                           struct nm_mpi_communicator_s*p_comm);
 
void nm_mpi_request_free(nm_mpi_request_t*req);
 
nm_mpi_request_t*nm_mpi_request_get(MPI_Request req_id);
 
int nm_mpi_request_test(nm_mpi_request_t*p_req);
 
int nm_mpi_request_wait(nm_mpi_request_t*p_req);
 
void nm_mpi_request_complete(nm_mpi_request_t*p_req, MPI_Request*request);
 
static inline void nm_mpi_request_set_datatype(nm_mpi_request_t*p_req, struct nm_mpi_datatype_s*p_datatype)
{
  p_req->p_datatype = p_datatype;
  if(p_datatype != NULL)
    nm_mpi_datatype_ref_inc(p_datatype, p_req);
}
 
static inline void nm_mpi_request_add_datatype2(nm_mpi_request_t*p_req, struct nm_mpi_datatype_s*p_datatype2)
{
  if(p_datatype2 != NULL)
    {
      assert(p_req != NULL);
      assert(p_req->p_datatype2 == NULL);
      p_req->p_datatype2 = p_datatype2;
      nm_mpi_datatype_ref_inc(p_datatype2, p_req);
    }
}
 
void nm_mpi_file_request_status_update(nm_mpi_request_t*p_req, struct nm_mpi_status_s*p_status);
 
int nm_mpi_file_request_test(nm_mpi_request_t*p_req);
 
int nm_mpi_file_request_wait(nm_mpi_request_t*p_req);
 
 
/* ** Send/recv/status functions *************************** */
 
int nm_mpi_isend_init(nm_mpi_request_t *p_req, int dest, nm_mpi_communicator_t *p_comm);
 
int nm_mpi_isend_start(nm_mpi_request_t *p_req);
 
int nm_mpi_isend(nm_mpi_request_t *p_req, int dest, nm_mpi_communicator_t *p_comm);
 
int nm_mpi_irecv_init(nm_mpi_request_t *p_req, int source, nm_mpi_communicator_t *p_comm);
 
int nm_mpi_irecv_start(nm_mpi_request_t *p_req);
 
int nm_mpi_irecv(nm_mpi_request_t *p_req, int source, nm_mpi_communicator_t *p_comm);
 
 
/* ** Collectives ****************************************** */
 
nm_mpi_request_t*nm_mpi_coll_isend(const void*buffer, nm_mpi_count_t count, nm_mpi_datatype_t*p_datatype, int dest, nm_tag_t tag, nm_mpi_communicator_t*p_comm);
 
nm_mpi_request_t*nm_mpi_coll_irecv(void*buffer, nm_mpi_count_t count, nm_mpi_datatype_t*p_datatype, int source, nm_tag_t tag, nm_mpi_communicator_t*p_comm);
 
void nm_mpi_coll_wait(nm_mpi_request_t*p_req);
 
int nm_mpi_coll_start(nm_mpi_request_t*p_req);
 
void nm_mpi_coll_free(nm_mpi_request_t*p_req);
 
 
struct nm_coll_req_s*nm_mpi_coll_ireduce_init(nm_mpi_communicator_t*p_comm, int root,
                                              nm_mpi_count_t count, nm_mpi_datatype_t*p_datatype,
                                              const void*sendbuf, void*recvbuf,
                                              nm_tag_t tag, nm_mpi_operator_t*p_operator,
                                              nm_coll_req_notifier_t p_notify, void*ref);
void nm_mpi_coll_ireduce_start(struct nm_coll_req_s*p_coll_req);
 
int nm_mpi_coll_reduce(nm_mpi_communicator_t*p_comm, int root,
                       nm_mpi_count_t count, nm_mpi_datatype_t*p_datatype,
                       const void*sendbuf, void*recvbuf,
                       nm_tag_t tag, nm_mpi_operator_t*p_operator);
 
/* ** iallreduce */
 
struct nm_coll_req_s*nm_mpi_coll_iallreduce(nm_mpi_communicator_t*p_comm,
                                            nm_mpi_count_t count, nm_mpi_datatype_t*p_datatype,
                                            const void*sendbuf, void*recvbuf,
                                            nm_tag_t tag, nm_mpi_operator_t*p_operator);
 
/* ** ialltoall */
 
struct nm_coll_req_s*nm_mpi_coll_ialltoall(nm_mpi_communicator_t*p_comm,
                                           const void*sendbuf, nm_mpi_count_t sendcount, nm_mpi_datatype_t*p_send_datatype,
                                           void*recvbuf, nm_mpi_count_t recvcount, nm_mpi_datatype_t*p_recv_datatype);
 
/* ** ialltoallv */
 
struct nm_coll_req_s*nm_mpi_coll_ialltoallv(nm_mpi_communicator_t*p_comm,
                                            const void*sendbuf, const nm_mpi_count_t*sendcounts, const nm_mpi_aint_t*sdispls, nm_mpi_datatype_t*p_send_datatype,
                                            void*recvbuf,  const nm_mpi_count_t*recvcounts, const nm_mpi_aint_t*rdispls, nm_mpi_datatype_t*p_recv_datatype);
 
/* ** iallgather */
 
struct nm_coll_req_s*nm_mpi_coll_iallgather(nm_mpi_communicator_t*p_comm,
                                            const void*sendbuf, nm_mpi_count_t sendcount, nm_mpi_datatype_t*p_send_datatype,
                                            void*recvbuf, nm_mpi_count_t recvcount, nm_mpi_datatype_t*p_recv_datatype);
 
 
/* ** Operators ******************************************** */
 
nm_mpi_operator_t*nm_mpi_operator_get(MPI_Op op);
 
void nm_mpi_operator_apply(void*invec, void*outvec, nm_mpi_count_t count, nm_mpi_datatype_t*p_datatype,
                           nm_mpi_operator_t*p_operator);
 
/* ** Communicator operations ****************************** */
 
nm_mpi_communicator_t*nm_mpi_communicator_alloc(nm_comm_t p_nm_comm, struct nm_mpi_errhandler_s*p_errhandler,
                                                enum nm_mpi_communicator_kind_e kind);
 
nm_mpi_communicator_t*nm_mpi_communicator_get(MPI_Comm comm);
 
#define nm_mpi_communicator_ref_inc(P_COMM, P_HOLDER) nm_refcount_inc(&(P_COMM)->refcount, (P_HOLDER));
 
void nm_mpi_communicator_ref_dec(nm_mpi_communicator_t*p_comm, const void*p_holder);
 
 
/* ** Attributes ******************************************* */
 
struct nm_mpi_keyval_s*nm_mpi_keyval_new(void);
 
void nm_mpi_keyval_delete(struct nm_mpi_keyval_s*p_keyval);
 
struct nm_mpi_keyval_s*nm_mpi_keyval_get(int id);
 
void nm_mpi_attrs_create(struct nm_mpi_attrs_s*p_attrs, int id);
 
int nm_mpi_attrs_copy(struct nm_mpi_attrs_s*p_old_attrs, struct nm_mpi_attrs_s*p_new_attrs);
 
void nm_mpi_attrs_destroy(struct nm_mpi_attrs_s*p_old_attrs);
 
int nm_mpi_attr_put(struct nm_mpi_attrs_s*p_attrs, struct nm_mpi_keyval_s*p_keyval, void*attr_value);
 
void nm_mpi_attr_get(struct nm_mpi_attrs_s*p_attrs, struct nm_mpi_keyval_s*p_keyval, void**p_attr_value, int*flag);
 
int nm_mpi_attr_delete(struct nm_mpi_attrs_s*p_attrs, struct nm_mpi_keyval_s*p_keyval);
 
 
/* ** Window operations *************************************/
 
nm_mpi_window_t*nm_mpi_window_get(MPI_Win win);
 
static inline uint16_t nm_mpi_win_get_next_seq(nm_mpi_window_t*p_win)
{
  return __sync_fetch_and_add(&p_win->next_seq, 1);
}
 
static inline int nm_mpi_win_completed_epoch(nm_mpi_win_epoch_t*p_epoch)
{
  int ret = __sync_bool_compare_and_swap(&p_epoch->nmsg, p_epoch->completed, p_epoch->nmsg);
  return ret;
}
 
static inline int nm_mpi_win_is_ready(nm_mpi_win_epoch_t*p_epoch)
{
  int ret = NM_MPI_WIN_UNUSED != p_epoch->mode;
  return ret;
}
 
static inline int nm_mpi_win_valid_assert(int assert)
{
  int ret = assert >= 0 && assert <= (MPI_MODE_NOCHECK | MPI_MODE_NOSTORE | MPI_MODE_NOPUT
                                      | MPI_MODE_NOPRECEDE | MPI_MODE_NOSUCCEED);
  return ret;
}
 
int nm_mpi_win_addr_is_valid(void*base, nm_mpi_aint_t extent, nm_mpi_window_t*p_win);
 
int nm_mpi_win_send_datatype(nm_mpi_datatype_t*p_datatype, int target_rank, nm_mpi_window_t*p_win);
 
 
/* ** RMA operations *************************************** */
 
static inline int nm_mpi_rma_tag_to_win(nm_tag_t tag)
{
  return (tag & NM_MPI_TAG_PRIVATE_RMA_MASK_WIN) >> 32;
}
 
static inline nm_tag_t nm_mpi_rma_win_to_tag(int win_id)
{
  return ((nm_tag_t)win_id) << 32;
}
 
static inline uint16_t nm_mpi_rma_tag_to_seq(nm_tag_t tag)
{
  return (tag & NM_MPI_TAG_PRIVATE_RMA_MASK_SEQ) >> 8;
}
 
static inline nm_tag_t nm_mpi_rma_seq_to_tag(uint16_t seq)
{
  return (nm_tag_t)((((nm_tag_t)seq) << 8) & 0xFFFF00);
}
 
static inline MPI_Op nm_mpi_rma_tag_to_mpi_op(nm_tag_t tag)
{
  return (tag & 0xF0) >> 4;
}
 
static inline nm_tag_t nm_mpi_rma_mpi_op_to_tag(MPI_Op op)
{
  return (nm_tag_t)((((nm_tag_t)op) << 4) & 0xF0);
}
 
static inline nm_tag_t nm_mpi_rma_create_usertag(uint16_t seq, int user_tag)
{
  return nm_mpi_rma_seq_to_tag(seq) + (((nm_tag_t)user_tag) & 0xFC0000FF);
}
 
static inline nm_tag_t nm_mpi_rma_create_tag(int win_id,  uint16_t seq_num, int user_tag)
{
  return nm_mpi_rma_win_to_tag(win_id) + nm_mpi_rma_create_usertag(seq_num, user_tag);
}
 
/* ** Keyval operations *************************************** */
 
int nm_mpi_comm_create_keyval_fort(nm_mpi_copy_subroutine_t*comm_copy_attr_fn,
                                   nm_mpi_delete_subroutine_t*comm_delete_attr_fn,
                                   int*comm_keyval, void*extra_state);
 
int nm_mpi_type_create_keyval_fort(nm_mpi_copy_subroutine_t*type_copy_attr_fn,
                                   nm_mpi_delete_subroutine_t*type_delete_attr_fn,
                                   int*type_keyval, void*extra_state);
 
int nm_mpi_win_create_keyval_fort(nm_mpi_copy_subroutine_t*win_copy_attr_fn,
                                  nm_mpi_delete_subroutine_t*win_delete_attr_fn,
                                  int*win_keyval, void*extra_state);
 
/* ** Threading ******************************************** */
 
int nm_mpi_thread_level_get(int required);
 
 
/* ** Large counts ***************************************** */
 
static inline nm_mpi_count_t*nm_mpi_array_count_from_int(nm_mpi_count_t n, const int*array_int)
{
  if(array_int == NULL)
    return NULL;
  nm_mpi_count_t*array_count = padico_malloc(sizeof(nm_mpi_count_t) * n);
  nm_mpi_count_t i;
  for(i = 0; i < n; i++)
    {
      array_count[i] = array_int[i];
    }
  return array_count;
}
 
static inline nm_mpi_aint_t*nm_mpi_array_aint_from_int(nm_mpi_count_t n, const int*array_int)
{
  if(array_int == NULL)
    return NULL;
  nm_mpi_aint_t*array_aint = padico_malloc(sizeof(nm_mpi_aint_t) * n);
  nm_mpi_count_t i;
  for(i = 0; i < n; i++)
    {
      array_aint[i] = array_int[i];
    }
  return array_aint;
}
 
static inline nm_mpi_count_t*nm_mpi_array_count_from_aint(nm_mpi_count_t n, const nm_mpi_aint_t*array_aint)
{
  if(array_aint == NULL)
    return NULL;
  nm_mpi_count_t*array_count = padico_malloc(sizeof(nm_mpi_count_t) * n);
  nm_mpi_count_t i;
  for(i = 0; i < n; i++)
    {
      array_count[i] = array_aint[i];
    }
  return array_count;
}
 
static inline nm_mpi_count_t*nm_mpi_array_count_copy(nm_mpi_count_t n, const nm_mpi_count_t*array_count)
{
  if(array_count == NULL)
    return NULL;
  nm_mpi_count_t*copy_array_count = padico_malloc(sizeof(nm_mpi_count_t) * n);
  nm_mpi_count_t i;
  for(i = 0; i < n; i++)
    {
      copy_array_count[i] = array_count[i];
    }
  return copy_array_count;
}
 
static inline nm_mpi_aint_t*nm_mpi_array_aint_copy(nm_mpi_count_t n, const nm_mpi_aint_t*array_aint)
{
  if(array_aint == NULL)
    return NULL;
  nm_mpi_aint_t*copy_array_aint = padico_malloc(sizeof(nm_mpi_aint_t) * n);
  nm_mpi_count_t i;
  for(i = 0; i < n; i++)
    {
      copy_array_aint[i] = array_aint[i];
    }
  return copy_array_aint;
}
 
static inline void nm_mpi_array_count_free(nm_mpi_count_t*array_count)
{
  if(array_count != NULL)
    padico_free(array_count);
}
 
static inline void nm_mpi_array_aint_free(nm_mpi_aint_t*array_aint)
{
  if(array_aint != NULL)
    padico_free(array_aint);
}
 
 
#endif /* NM_MPI_PRIVATE_H */