nm_ibverbs.h

Go to the documentation of this file.

/*
 * NewMadeleine
 * Copyright (C) 2010-2026 (see AUTHORS file)
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 */
 
#ifndef NM_IBVERBS_H
#define NM_IBVERBS_H
 
#include <nm_minidriver.h>
#include <infiniband/verbs.h>
#include <nm_public.h>
#include <nm_private.h>
#ifdef NMAD_PADICOTM
#include <Padico/SysIO.h>
#endif /* NMAD_PADICOTM*/
 
/* *** global IB parameters ******************************** */
 
#define NM_IBVERBS_UD_DEPTH     8192   
#define NM_IBVERBS_TX_DEPTH     256    
#define NM_IBVERBS_RX_DEPTH     1024   
#define NM_IBVERBS_SRQ_DEPTH    16384  
#define NM_IBVERBS_ATOMIC_DEPTH 4      
#define NM_IBVERBS_MAX_SG_SQ    16     
#define NM_IBVERBS_MAX_SG_RQ    16     
#define NM_IBVERBS_MAX_INLINE   256
#define NM_IBVERBS_DEFAULT_MTU  IBV_MTU_1024
 
#define NM_IBVERBS_TIMEOUT_CONNECT (1000 * 1000) /* 1 s */
 
#define NM_IBVERBS_TIMEOUT_WARN (8 * 1000 * 1000) /* 8 s */
 
#define NM_IBVERBS_TIMEOUT_READY (16 * 1000 * 1000) /* 16 s */
 
#define GRH_OFFSET 40 /* 40 */
 
uint32_t nm_ibverbs_checksum(const char*data, nm_len_t len);
uint32_t nm_ibverbs_memcpy_and_checksum(void*_dest, const void*_src, nm_len_t len);
int nm_ibverbs_checksum_enabled(void);
uint32_t nm_ibverbs_copy_from_and_checksum(void*dest, nm_data_slicer_t*p_slicer, const void*src, nm_len_t offset, nm_len_t len);
uint32_t nm_ibverbs_copy_to_and_checksum(const void*src, nm_data_slicer_t*p_slicer, void*dest, nm_len_t offset, nm_len_t len);
 
extern int nm_ibverbs_alignment;
extern int nm_ibverbs_memalign;
 
enum nm_ibverbs_wrid_e
  {
    _NM_IBVERBS_WRID_NONE = 0,
    NM_IBVERBS_WRID_ACK,    
    NM_IBVERBS_WRID_RDMA,   
    NM_IBVERBS_WRID_RDV,    
    NM_IBVERBS_WRID_DATA,   
    NM_IBVERBS_WRID_HEADER, 
    NM_IBVERBS_WRID_PACKET, 
    NM_IBVERBS_WRID_SEG,    
    NM_IBVERBS_WRID_RTR,
    NM_IBVERBS_WRID_RECV,
    NM_IBVERBS_WRID_SEND,
    _NM_IBVERBS_WRID_MAX
  };
enum nm_ibverbs_wrid_ud_e
  {
    NM_IBVERBS_WRID_UD_SEND = _NM_IBVERBS_WRID_MAX + 1,
    NM_IBVERBS_WRID_UD_RECV,
    NM_IBVERBS_WRID_UD_RECV_POOL_FIRST = 32,
    NM_IBVERBS_WRID_UD_RECV_POOL_LAST  = 32 + NM_IBVERBS_UD_DEPTH,
  };
 
struct nm_ibverbs_segment_s
{
  uint64_t raddr;  
  uint32_t rkey;   
};
 
#define NM_IBVERBS_SEGMENT_NULL ((struct nm_ibverbs_segment_s){ .raddr = 0, .rkey = 0 })
static inline int nm_ibverbs_segment_isnull(const struct nm_ibverbs_segment_s*p_seg)
{
  return (p_seg->raddr == 0);
}
 
struct nm_ibverbs_addr_s
{
  uint16_t lid;
  uint16_t cnx_id;
  uint32_t qpn;
  uint32_t psn;
  union ibv_gid gid;
};
 
/* ** hash functions for QPN */
 
static inline int nm_ibverbs_qpn_eq(const uint32_t*p_int1, const uint32_t*p_int2)
{
  return (*p_int1 == *p_int2);
}
static inline uint32_t nm_ibverbs_qpn_hash(const uint32_t*p_int)
{
  return *p_int;
}
 
/* ** hash functions for addresses */
 
static inline int nm_ibverbs_addr_eq(const struct nm_ibverbs_addr_s*p_addr1, const struct nm_ibverbs_addr_s*p_addr2)
{
  return
    (p_addr1->lid == p_addr2->lid) &&
    (p_addr1->qpn == p_addr2->qpn) &&
    (p_addr1->psn == p_addr2->psn) &&
    (p_addr1->cnx_id == p_addr2->cnx_id);
}
static inline uint32_t nm_ibverbs_addr_hash(const struct nm_ibverbs_addr_s*p_addr)
{
  return puk_hash_default((const void*)p_addr, sizeof(struct nm_ibverbs_addr_s));
}
 
struct nm_ibverbs_cnx_addr_s
{
  struct nm_ibverbs_addr_s addr;
  struct nm_ibverbs_segment_s segment;
};
 
struct nm_ibverbs_hca_s
{
  const char*device;
  int port;
  struct ibv_device*ib_dev;   
  uint16_t lid;               
  union ibv_gid gid;          
  int gidx                    ;
  struct ibv_context*context; 
  struct ibv_pd*pd;           
  struct
  {
    int max_qp;               
    int max_qp_wr;            
    int max_cq;               
    int max_cqe;              
    int max_mr;               
    int max_srq;              
    uint64_t max_mr_size;     
    uint64_t page_size_cap;   
    uint64_t max_msg_size;    
    int data_rate;
  } ib_caps;
  struct
  {
#ifdef NMAD_PADICOTM
    padico_sysio_t async_io;    
#endif /* NMAD_PADICOTM */
    int srq_limit_reached;      
  } async_events;
  nm_len_t size_mem_reg;        
  long n_mem_reg;               
  void (*p_reg_back_pressure)(void*, nm_len_t); 
  void*_reg_back_pressure_status; 
};
 
 
/* ** UD messaging ***************************************** */
 
struct nm_ibverbs_ud_ep_s
{
  struct nm_ibverbs_hca_s*p_hca;       
  struct nm_ibverbs_addr_s local_addr; 
  struct ibv_qp*qp;                    
  struct ibv_cq*of_cq;                 
  struct ibv_cq*if_cq;                 
};
 
struct nm_ibverbs_ud_recv_s
{
  struct nm_ibverbs_ud_ep_s*p_ud_ep;
  struct ibv_mr*mr;
};
 
struct nm_ibverbs_ud_recv_pool_s
{
  struct nm_ibverbs_ud_ep_s*p_ud_ep;
  struct ibv_mr*mr;
  void*buf;
  size_t size;
  int depth;
};
 
struct nm_ibverbs_ud_send_s
{
  struct nm_ibverbs_ud_ep_s*p_ud_ep;
  struct ibv_ah*ah;
  struct ibv_mr*mr;
};
 
 
/* ** connection establishment ***************************** */
 
struct nm_ibverbs_cnx_addr_msg_s
{
  struct nm_ibverbs_addr_s ud_addr;       
  struct nm_ibverbs_cnx_addr_s cnx_addr;  
  enum nm_ibverbs_cnx_addr_state_e
    {
      NM_IBVERBS_CNX_ADDR_NONE      = 0,  
      NM_IBVERBS_CNX_ADDR_OK        = 1,  
      NM_IBVERBS_CNX_ADDR_CONNECTED = 2   
    } state;                              
  int timeout;                            
  uint64_t oob_data;                      
};
 
struct nm_ibverbs_cnx_connect_packet_s
{
  char grh_header[GRH_OFFSET];
  struct nm_ibverbs_cnx_addr_msg_s addr_msg;
};
 
PUK_LIST_TYPE(nm_ibverbs_cnx_addr_entry,
              struct nm_ibverbs_cnx_addr_msg_s msg;  
              enum nm_ibverbs_cnx_addr_state_e local_state; 
              int local_timeout;                     
              uint64_t local_oob;                    
              struct nm_ibverbs_cnx_s*p_cnx;         
              double t_timeout;                      
              );
 
static void nm_ibverbs_cnx_addr_entry_destructor(struct nm_ibverbs_addr_s*p_addr, struct nm_ibverbs_cnx_addr_entry_s*p_entry)
{
  padico_free(p_entry);
}
 
PUK_HASHTABLE_TYPE(nm_ibverbs_cnx_addr_entry, struct nm_ibverbs_addr_s*, struct nm_ibverbs_cnx_addr_entry_s*,
                   &nm_ibverbs_addr_hash, &nm_ibverbs_addr_eq, &nm_ibverbs_cnx_addr_entry_destructor);
 
PUK_HASHTABLE_TYPE(nm_ibverbs_cnx_qpn, const uint32_t*, struct nm_ibverbs_cnx_s*,
                   &nm_ibverbs_qpn_hash, &nm_ibverbs_qpn_eq, NULL);
 
 
/* ** public structures ************************************ */
 
struct nm_ibverbs_context_s
{
  struct nm_ibverbs_hca_s hca;            
  struct
  {
    nm_spinlock_t lock;                   
    struct nm_ibverbs_ud_ep_s udep;       
    struct nm_ibverbs_cnx_addr_entry_hashtable_s addr_hash; 
    struct nm_ibverbs_cnx_qpn_hashtable_s cnx_by_qpn;       
    struct nm_ibverbs_cnx_addr_entry_list_s pending_entries; 
    struct nm_ibverbs_ud_recv_pool_s recv_pool;
    struct nm_ibverbs_cnx_connect_packet_s recv_packets[NM_IBVERBS_UD_DEPTH];
  } connect;
  struct
  {
    void*p_wide_url;                         
    int session_size;
    uint32_t rank;
    struct nm_ibverbs_cnx_s**pp_cached_cnxs; 
  } vector_connect;
  struct
  {
    struct ibv_srq*p_srq;                 
    struct ibv_cq*srq_cq;                 
    struct ibv_comp_channel*comp_channel; 
    int cancel_fd[2];                     
    volatile int waiting;                 
  } srq;
  struct nm_ibverbs_opts_s
  {
    int lazy_connect;                     
    int use_srq;                          
    int use_comp_channel;                 
    int mtu;                              
  } ib_opts;
};
 
struct nm_ibverbs_cnx_s
{
  struct nm_ibverbs_cnx_addr_entry_s*p_addr_entry; 
  struct nm_ibverbs_cnx_addr_s local_addr;
  struct nm_ibverbs_cnx_addr_s remote_addr;
  struct ibv_qp*qp;       
  struct ibv_cq*of_cq;    
  struct ibv_cq*if_cq;    
  int max_inline;         
  struct nm_ibverbs_context_s*p_ibverbs_context;
  struct
  {
    int total;
    int wrids[_NM_IBVERBS_WRID_MAX];
  } pending;              
  struct
  {
    int remote_rank;
  } vector_connect;        
};
 
struct nm_ibverbs_iface_s
{
  struct nm_ibverbs_context_s*(*context_new)(puk_context_t, const struct nm_ibverbs_opts_s*);
};
PUK_IFACE_TYPE(NewMad_ibverbs, struct nm_ibverbs_iface_s);
 
/* ** HCA ************************************************** */
 
void nm_ibverbs_hca_open(struct nm_ibverbs_hca_s*p_hca, const char*device, int port, int gidx);
 
void nm_ibverbs_hca_close(struct nm_ibverbs_hca_s*p_hca);
 
/* ** IB cnx management ************************************ */
 
struct nm_ibverbs_cnx_s*nm_ibverbs_cnx_create(struct nm_ibverbs_context_s*p_ibverbs_context, int cnx_id);
 
void nm_ibverbs_cnx_close(struct nm_ibverbs_cnx_s*p_ibverbs_cnx);
 
void nm_ibverbs_cnx_connect_async(struct nm_ibverbs_cnx_s*p_ibverbs_cnx, const void*p_url, size_t urlsize);
 
void nm_ibverbs_cnx_connect_wait(struct nm_ibverbs_cnx_s*p_ibverbs_cnx);
 
/* ** vector connect *************************************** */
 
struct nm_ibverbs_cnx_s*nm_ibverbs_cnx_vector_connect(struct nm_ibverbs_context_s*p_ibverbs_context,
                                                      const void*p_remote_url, size_t url_size);
 
int nm_ibverbs_cnx_vector_enabled(void);
 
/* ** lazy connect ***************************************** */
 
int nm_ibverbs_cnx_lazy_enabled(void);
 
void nm_ibverbs_cnx_lazy_create(struct nm_ibverbs_context_s*p_ibverbs_context, struct nm_ibverbs_cnx_s**pp_ibverbs_cnx,
                                const void*p_remote_url, size_t url_size, int cnx_id);
 
static inline int nm_ibverbs_cnx_lazy_ready(struct nm_ibverbs_cnx_s*p_ibverbs_cnx)
{
  return ( (p_ibverbs_cnx != NULL) &&
           (p_ibverbs_cnx->p_addr_entry != NULL) &&
           (p_ibverbs_cnx->p_addr_entry->msg.state == NM_IBVERBS_CNX_ADDR_CONNECTED) &&
           (p_ibverbs_cnx->p_addr_entry->local_state == NM_IBVERBS_CNX_ADDR_CONNECTED) );
}
 
void nm_ibverbs_cnx_lazy_send_oob(struct nm_ibverbs_cnx_s*p_ibverbs_cnx, uint64_t oob);
 
static inline uint64_t nm_ibverbs_cnx_lazy_get_oob(struct nm_ibverbs_cnx_s*p_ibverbs_cnx)
{
  assert(nm_ibverbs_cnx_lazy_ready(p_ibverbs_cnx));
  return p_ibverbs_cnx->p_addr_entry->msg.oob_data;
}
 
void nm_ibverbs_cnx_lazy_progress(struct nm_ibverbs_context_s*p_ibverbs_context);
 
static inline
struct nm_ibverbs_cnx_s*nm_ibverbs_cnx_lazy_by_qpn(struct nm_ibverbs_context_s*p_ibverbs_context, uint32_t qpn)
{
  struct nm_ibverbs_cnx_s*p_cnx = nm_ibverbs_cnx_qpn_hashtable_lookup(&p_ibverbs_context->connect.cnx_by_qpn, &qpn);
  return p_cnx;
}
 
/* ** IB context ******************************************* */
 
struct nm_ibverbs_context_s*nm_ibverbs_context_resolve_opts(puk_context_t p_context, const struct nm_ibverbs_opts_s*p_opts);
 
static inline void nm_ibverbs_context_default_opts(struct nm_ibverbs_opts_s*p_opts)
{
  p_opts->lazy_connect = 0;
  p_opts->use_srq = 0;
  p_opts->use_comp_channel = 0;
  p_opts->mtu = IBV_MTU_1024;
}
 
static inline struct nm_ibverbs_context_s*nm_ibverbs_context_resolve(puk_context_t p_context)
{
  struct nm_ibverbs_opts_s opts = { 0 };
  nm_ibverbs_context_default_opts(&opts);
  return nm_ibverbs_context_resolve_opts(p_context, &opts);
}
 
int nm_ibverbs_context_wait_event(struct nm_ibverbs_context_s*p_ibverbs_context);
 
void nm_ibverbs_context_cancel_wait(struct nm_ibverbs_context_s*p_ibverbs_context);
 
void nm_ibverbs_context_get_profile(struct nm_ibverbs_context_s*p_ibverbs_context, struct nm_drv_profile_s*p_profile);
 
void nm_ibverbs_context_get_url(struct nm_ibverbs_context_s*p_ibverbs_context, const void**pp_url, size_t*p_urlsize);
 
void nm_ibverbs_context_get_wide_url(struct nm_ibverbs_context_s*p_ibverbs_context, int session_size, int rank,
                                     const void**pp_url, size_t*p_urlsize);
 
void nm_ibverbs_context_delete(struct nm_ibverbs_context_s*p_ibverbs_context);
 
void nm_ibverbs_context_set_srq_limit(struct nm_ibverbs_context_s*p_ibverbs_context, int srq_limit);
 
struct ibv_mr*nm_ibverbs_reg_mr(struct nm_ibverbs_context_s*p_ibverbs_context, void*buf, size_t size,
                                struct nm_ibverbs_segment_s*p_segment);
 
int nm_ibverbs_dereg_mr(struct nm_ibverbs_context_s*p_ibverbs_context, struct ibv_mr*p_mr);
 
/* ** RDMA toolbox ***************************************** */
 
 
static const char*const nm_ibverbs_status_strings[] =
  {
    [IBV_WC_SUCCESS]            = "success",
    [IBV_WC_LOC_LEN_ERR]        = "local length error",
    [IBV_WC_LOC_QP_OP_ERR]      = "local QP op error",
    [IBV_WC_LOC_EEC_OP_ERR]     = "local EEC op error",
    [IBV_WC_LOC_PROT_ERR]       = "local protection error",
    [IBV_WC_WR_FLUSH_ERR]       = "write flush error",
    [IBV_WC_MW_BIND_ERR]        = "MW bind error",
    [IBV_WC_BAD_RESP_ERR]       = "bad response error",
    [IBV_WC_LOC_ACCESS_ERR]     = "local access error",
    [IBV_WC_REM_INV_REQ_ERR]    = "remote invalid request error",
    [IBV_WC_REM_ACCESS_ERR]     = "remote access error",
    [IBV_WC_REM_OP_ERR]         = "remote op error",
    [IBV_WC_RETRY_EXC_ERR]      = "retry exceded error",
    [IBV_WC_RNR_RETRY_EXC_ERR]  = "RNR retry exceded error",
    [IBV_WC_LOC_RDD_VIOL_ERR]   = "local RDD violation error",
    [IBV_WC_REM_INV_RD_REQ_ERR] = "remote invalid read request error",
    [IBV_WC_REM_ABORT_ERR]      = "remote abort error",
    [IBV_WC_INV_EECN_ERR]       = "invalid EECN error",
    [IBV_WC_INV_EEC_STATE_ERR]  = "invalid EEC state error",
    [IBV_WC_FATAL_ERR]          = "fatal error",
    [IBV_WC_RESP_TIMEOUT_ERR]   = "response timeout error",
    [IBV_WC_GENERAL_ERR]        = "general error"
  };
 
static const int nm_ibverbs_mtu_values[] =
  {
   [ IBV_MTU_256 ]  = 256,
   [ IBV_MTU_512 ]  = 512,
   [ IBV_MTU_1024 ] = 1024,
   [ IBV_MTU_2048 ] = 2048,
   [ IBV_MTU_4096 ] = 4096
  };
 
int nm_ibverbs_do_rdma(struct nm_ibverbs_cnx_s*__restrict__ p_ibverbs_cnx,
                       const void*__restrict__ buf, int size, uint64_t raddr,
                       int opcode, int flags, uint32_t lkey, uint32_t rkey, uint64_t wrid)
  __attribute__ ((warn_unused_result));
 
 
int nm_ibverbs_rdma_send(struct nm_ibverbs_cnx_s*p_ibverbs_cnx, int size,
                         const void*__restrict__ ptr,
                         const void*__restrict__ _raddr,
                         const void*__restrict__ _lbase,
                         const struct nm_ibverbs_segment_s*p_seg,
                         const struct ibv_mr*__restrict__ mr,
                         int wrid)
  __attribute__ ((warn_unused_result));
 
int nm_ibverbs_rdma_poll_inout(struct nm_ibverbs_cnx_s*__restrict__ p_ibverbs_cnx, int inout)
  __attribute__ ((warn_unused_result));
 
static inline int nm_ibverbs_rdma_poll_send(struct nm_ibverbs_cnx_s*__restrict__ p_ibverbs_cnx)
  __attribute__ ((warn_unused_result));
static inline int nm_ibverbs_rdma_poll_send(struct nm_ibverbs_cnx_s*__restrict__ p_ibverbs_cnx)
{
  return nm_ibverbs_rdma_poll_inout(p_ibverbs_cnx, 0);
}
 
static inline int nm_ibverbs_rdma_poll_recv(struct nm_ibverbs_cnx_s*__restrict__ p_ibverbs_cnx)
  __attribute__ ((warn_unused_result));
static inline int nm_ibverbs_rdma_poll_recv(struct nm_ibverbs_cnx_s*__restrict__ p_ibverbs_cnx)
{
  return nm_ibverbs_rdma_poll_inout(p_ibverbs_cnx, 1);
}
 
 
static inline int nm_ibverbs_recv_poll(struct nm_ibverbs_cnx_s*__restrict__ p_ibverbs_cnx, uint64_t wrid)
  __attribute__ ((warn_unused_result));
static inline int nm_ibverbs_recv_poll(struct nm_ibverbs_cnx_s*__restrict__ p_ibverbs_cnx, uint64_t wrid)
{
  int rc = nm_ibverbs_rdma_poll_recv(p_ibverbs_cnx);
  if(rc != NM_ESUCCESS)
    return rc;
  if(p_ibverbs_cnx->pending.wrids[wrid])
    return -NM_EAGAIN;
  return NM_ESUCCESS;
}
 
static inline int nm_ibverbs_send_poll(struct nm_ibverbs_cnx_s*__restrict__ p_ibverbs_cnx, uint64_t wrid)
  __attribute__ ((warn_unused_result));
static inline int nm_ibverbs_send_poll(struct nm_ibverbs_cnx_s*__restrict__ p_ibverbs_cnx, uint64_t wrid)
{
  int rc = nm_ibverbs_rdma_poll_send(p_ibverbs_cnx);
  if(rc != NM_ESUCCESS)
    return rc;
  if(p_ibverbs_cnx->pending.wrids[wrid])
    return -NM_EAGAIN;
  return NM_ESUCCESS;
}
 
static inline int nm_ibverbs_send_flush(struct nm_ibverbs_cnx_s*__restrict__ p_ibverbs_cnx, uint64_t wrid)
  __attribute__ ((warn_unused_result));
static inline int nm_ibverbs_send_flush(struct nm_ibverbs_cnx_s*__restrict__ p_ibverbs_cnx, uint64_t wrid)
{
  while(p_ibverbs_cnx->pending.wrids[wrid])
    {
      int rc = nm_ibverbs_rdma_poll_send(p_ibverbs_cnx);
      if(rc != NM_ESUCCESS)
        return rc;
    }
  return NM_ESUCCESS;
}
 
static inline int nm_ibverbs_send_flushn_total(struct nm_ibverbs_cnx_s*__restrict__ p_ibverbs_cnx, int n)
  __attribute__ ((warn_unused_result));
static inline int nm_ibverbs_send_flushn_total(struct nm_ibverbs_cnx_s*__restrict__ p_ibverbs_cnx, int n)
{
  while(p_ibverbs_cnx->pending.total > n)
    {
      int rc = nm_ibverbs_rdma_poll_send(p_ibverbs_cnx);
      if(rc != NM_ESUCCESS)
        return rc;
    }
  return NM_ESUCCESS;
}
 
static inline int nm_ibverbs_send_flushn(struct nm_ibverbs_cnx_s*__restrict__ p_ibverbs_cnx, int wrid, int n)
  __attribute__ ((warn_unused_result));
static inline int nm_ibverbs_send_flushn(struct nm_ibverbs_cnx_s*__restrict__ p_ibverbs_cnx, int wrid, int n)
{
  while(p_ibverbs_cnx->pending.wrids[wrid] > n)
    {
      int rc = nm_ibverbs_rdma_poll_send(p_ibverbs_cnx);
      if(rc != NM_ESUCCESS)
        return rc;
    }
  return NM_ESUCCESS;
}
 
/* ** IB UD ************************************************ */
 
void nm_ibverbs_ud_ep_open(struct nm_ibverbs_ud_ep_s*p_ud_ep, struct nm_ibverbs_hca_s*p_hca);
 
void nm_ibverbs_ud_ep_close(struct nm_ibverbs_ud_ep_s*p_ud_ep);
 
void nm_ibverbs_ud_send_post(struct nm_ibverbs_ud_send_s*p_send,
                             struct nm_ibverbs_ud_ep_s*p_ud_ep, void*buf, size_t size,
                             const struct nm_ibverbs_addr_s*p_remote_addr);
 
int nm_ibverbs_ud_send_poll(struct nm_ibverbs_ud_send_s*p_send);
 
void nm_ibverbs_ud_send(struct nm_ibverbs_ud_ep_s*p_ud_ep, void*buf, size_t size,
                        const struct nm_ibverbs_addr_s*p_remote_addr);
 
void nm_ibverbs_ud_recv_post(struct nm_ibverbs_ud_recv_s*p_recv,
                             struct nm_ibverbs_ud_ep_s*p_ud_ep, void*buf, size_t size);
 
int nm_ibverbs_ud_recv_poll(struct nm_ibverbs_ud_recv_s*p_recv);
 
void nm_ibverbs_ud_recv(struct nm_ibverbs_ud_ep_s*p_ud_ep, void*buf, size_t size);
 
void nm_ibverbs_ud_recv_pool_post(struct nm_ibverbs_ud_recv_pool_s*p_recv,
                                  struct nm_ibverbs_ud_ep_s*p_ud_ep, void*buf, size_t size, int depth);
 
int nm_ibverbs_ud_recv_pool_poll(struct nm_ibverbs_ud_recv_pool_s*p_recv);
 
void nm_ibverbs_ud_recv_pool_reload(struct nm_ibverbs_ud_recv_pool_s*p_recv, int i);
 
void nm_ibverbs_ud_recv_pool_delete(struct nm_ibverbs_ud_recv_pool_s*p_recv);
 
 
/* ** reg pool ********************************************* */
 
struct nm_ibverbs_regpool_entry_s
{
  void*p_ptr;
  struct ibv_mr*p_mr;
};
 
static inline void nm_ibverbs_regpool_entry_segment(const struct nm_ibverbs_regpool_entry_s*p_entry, struct nm_ibverbs_segment_s*p_seg)
{
  p_seg->raddr = (uintptr_t)p_entry->p_ptr;
  p_seg->rkey = p_entry->p_mr->rkey;
}
 
PUK_LFSTACK_TYPE(nm_ibverbs_regpool_block,
                 struct nm_ibverbs_regpool_entry_s entry;
                 );
 
PUK_LFSTACK_TYPE(nm_ibverbs_regpool_slice,
                 void*p_ptr;
                 struct ibv_mr*p_mr;
                 struct nm_ibverbs_regpool_block_lfstack_cell_s*p_cells;
                 );
 
struct nm_ibverbs_regpool_s
{
  struct nm_ibverbs_regpool_slice_lfstack_s slices;
  struct nm_ibverbs_regpool_block_lfstack_s blocks;
  nm_len_t entry_size;                              
  nm_len_t num_entries;                             
  struct nm_ibverbs_context_s*p_ibverbs_context;
#ifdef NMAD_PROFILE
  unsigned long long total_alloc;                   
  unsigned long max_alloc;                          
  unsigned long cur_alloc;
#endif /* NMAD_PROFILE */
};
 
void nm_ibverbs_regpool_init(struct nm_ibverbs_regpool_s*p_regpool, struct nm_ibverbs_context_s*p_ibverbs_context, nm_len_t entry_size, nm_len_t slice_size);
 
void nm_ibverbs_regpool_destroy(struct nm_ibverbs_regpool_s*p_regpool);
 
const struct nm_ibverbs_regpool_entry_s*nm_ibverbs_regpool_alloc_entry(struct nm_ibverbs_regpool_s*p_regpool);
 
void nm_ibverbs_regpool_free_entry(struct nm_ibverbs_regpool_s*p_regpool, const struct nm_ibverbs_regpool_entry_s*p_entry);
 
 
#endif /* NM_IBVERBS_H */