nm_core_interface.h

Go to the documentation of this file.

/*
 * NewMadeleine
 * Copyright (C) 2006-2026 (see AUTHORS file)
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 */
 
 
#ifndef NM_CORE_INTERFACE_H
#define NM_CORE_INTERFACE_H
 
#include <nm_public.h>
#include <nm_log.h>
#include <Padico/Puk.h>
#include <sys/uio.h>
 
#ifdef PIOMAN
#include <pioman.h>
#else
#include <pthread.h>
#endif
 
/* ** Core init ******************************************** */
 
typedef struct nm_core*nm_core_t;
 
puk_component_t nm_core_component_load(const char*entity, const char*name);
 
int nm_core_init(nm_core_t *pp_core);
 
int nm_core_set_strategy(nm_core_t p_core, puk_component_t strategy);
 
int nm_core_exit(nm_core_t p_core);
 
void nm_core_schedopt_disable(nm_core_t p_core);
 
extern struct nm_core_internal_s
{
  nm_core_t p_core_singleton;
} nm_core_internal;
 
__attribute__((pure))
static inline nm_core_t nm_core_get_singleton(void)
{
  return nm_core_internal.p_core_singleton;
}
 
void nm_trace_add_synchro_point(void);
 
 
/* ** Drivers ********************************************** */
 
typedef struct nm_drv_s*nm_drv_t;
 
PUK_VECT_TYPE(nm_drv, nm_drv_t);
 
int nm_core_driver_load_init(nm_core_t p_core, puk_component_t driver, nm_trk_kind_t kind,
                             nm_drv_t *pp_drv, const char**p_url);
 
 
/* ** Gates ************************************************ */
 
nm_gate_t nm_core_gate_new(nm_core_t p_core, nm_drv_vect_t*p_drvs);
void nm_core_gate_connect_async(nm_core_t p_core, nm_gate_t gate, nm_drv_t p_drv, nm_trk_id_t trk_id, const char*url);
void nm_core_gate_connect_wait(nm_core_t p_core, struct nm_trk_s*p_trk);
 
 
/* ** Threads ********************************************** */
 
enum nm_thread_level_e
  {
    NM_THREAD_SINGLE     = 0,
    NM_THREAD_FUNNELED   = 1,
    NM_THREAD_SERIALIZED = 2,
    NM_THREAD_MULTIPLE   = 3
  };
typedef enum nm_thread_level_e nm_thread_level_t;
 
nm_thread_level_t nm_core_get_thread_level(nm_core_t);
 
void nm_core_set_thread_level(nm_thread_level_t);
 
/* ** Progression ****************************************** */
 
int nm_schedule(nm_core_t p_core);
 
/* ** Status *********************************************** */
 
#ifdef PIOMAN
typedef piom_cond_value_t nm_status_t;
typedef piom_cond_t nm_cond_status_t;
#else /* PIOMAN */
typedef uint32_t nm_status_t;
typedef nm_status_t nm_cond_status_t;
#endif /* PIOMAN */
 
/* ** status and flags, used in pack/unpack requests and events */
 
#define NM_STATUS_NONE                     ((nm_status_t)0x00000000)
#define NM_STATUS_PACK_INIT                ((nm_status_t)0x00000001)
#define NM_STATUS_UNPACK_INIT              ((nm_status_t)0x00000002)
#define NM_STATUS_PACK_COMPLETED           ((nm_status_t)0x00000004)
#define NM_STATUS_UNPACK_COMPLETED         ((nm_status_t)0x00000008)
#define NM_STATUS_UNEXPECTED               ((nm_status_t)0x00000010)
#define NM_STATUS_UNPACK_CANCELLED         ((nm_status_t)0x00000020)
#define NM_STATUS_PACK_POSTED              ((nm_status_t)0x00000040)
#define NM_STATUS_UNPACK_POSTED            ((nm_status_t)0x00000080)
#define NM_STATUS_ACK_RECEIVED             ((nm_status_t)0x00000100)
#define NM_STATUS_UNPACK_DATA0             ((nm_status_t)0x00000200)
#define NM_STATUS_UNPACK_DATA_SIZE         ((nm_status_t)0x00000400)
#define NM_STATUS_FINALIZED                ((nm_status_t)0x00000800)
#define NM_STATUS_ERROR                    ((nm_status_t)0x00001000)
#define NM_STATUS_PACK_MSG_SIZE            ((nm_status_t)0x00002000)
#define NM_STATUS_UNPACK_PREFETCHED        ((nm_status_t)0x00004000)
 
#define NM_STATUS_MASK_FULL                ((nm_status_t)-1)
 
 
typedef uint32_t nm_req_flag_t;
 
#define NM_REQ_FLAG_NONE                   ((nm_req_flag_t)0x00000000)
#define NM_REQ_FLAG_PACK_SYNCHRONOUS       ((nm_req_flag_t)0x00001000)
#define NM_REQ_FLAG_PACK                   ((nm_req_flag_t)0x00002000)
#define NM_REQ_FLAG_UNPACK                 ((nm_req_flag_t)0x00004000)
#define NM_REQ_FLAG_UNPACK_DATA_INFO       ((nm_req_flag_t)0x00008000)
#define NM_REQ_FLAG_UNPACK_MATCHING_INFO   ((nm_req_flag_t)0x00010000)
#define NM_REQ_FLAG_UNPACK_PREFETCHING     ((nm_req_flag_t)0x00020000)
#define NM_REQ_FLAG_MATCHING_WILDCARD      ((nm_req_flag_t)0x00100000)
#define NM_REQ_FLAG_MATCHING_GATE          ((nm_req_flag_t)0x00200000)
#define NM_REQ_FLAG_MATCHING_TAG           ((nm_req_flag_t)0x00400000)
#define NM_REQ_FLAG_MATCHING_FULL          ((nm_req_flag_t)0x00800000)
#define NM_REQ_FLAG_FINALIZE_LATER         ((nm_req_flag_t)0x01000000)
#define NM_REQ_FLAG_UNPACK_PARTITIONED     ((nm_req_flag_t)0x02000000)
#define NM_REQ_FLAG_PACK_PARTITIONED       ((nm_req_flag_t)0x04000000)
 
typedef uint32_t nm_req_chunk_flag_t;
 
#define NM_REQ_CHUNK_FLAG_NONE             ((nm_req_chunk_flag_t)0x00000000)
#define NM_REQ_CHUNK_FLAG_SHORT            ((nm_req_chunk_flag_t)0x00020000)
#define NM_REQ_CHUNK_FLAG_USE_COPY         ((nm_req_chunk_flag_t)0x00080000)
#define NM_REQ_CHUNK_FLAG_DATA_ITERATOR    ((nm_req_chunk_flag_t)0x00100000)
 
/* ** tags ************************************************* */
 
typedef uint32_t nm_session_hash_t;
 
#define NM_CORE_TAG_HASH_FULL ((nm_session_hash_t)0xFFFFFFFF)
 
struct nm_core_tag_s
{
  nm_tag_t tag;               
  nm_session_hash_t hashcode; 
} __attribute__((packed));
 
typedef struct nm_core_tag_s nm_core_tag_t;
 
#define NM_CORE_TAG_MASK_FULL ((nm_core_tag_t){ .tag = NM_TAG_MASK_FULL, .hashcode = NM_CORE_TAG_HASH_FULL })
#define NM_CORE_TAG_NONE      ((nm_core_tag_t){ .tag = 0, .hashcode = 0x0 })
 
static inline nm_core_tag_t nm_core_tag_build(nm_session_hash_t hashcode, nm_tag_t tag)
{
  nm_core_tag_t core_tag;
  core_tag.tag = tag;
  core_tag.hashcode = hashcode;
  return core_tag;
}
static inline nm_tag_t nm_core_tag_get_tag(nm_core_tag_t core_tag)
{
  return core_tag.tag;
}
static inline nm_session_hash_t nm_core_tag_get_hashcode(nm_core_tag_t core_tag)
{
  return core_tag.hashcode;
}
 
/* ** Event notification *********************************** */
 
struct nm_core_event_s
{
  nm_status_t status; 
  nm_gate_t p_gate;
  nm_core_tag_t tag;
  nm_seq_t seq;
  nm_len_t len;
  struct nm_req_s*p_req; 
};
 
typedef void (*nm_core_event_notifier_t)(const struct nm_core_event_s*const event, void*ref);
 
struct nm_core_event_matching_s
{
  nm_gate_t p_gate;       
  nm_core_tag_t tag;      
  nm_core_tag_t tag_mask; 
};
 
struct nm_monitor_s
{
  nm_core_event_notifier_t p_notifier; 
  nm_status_t event_mask;              
  void*ref;                            
};
 
struct nm_core_monitor_s
{
  struct nm_monitor_s monitor;               
  struct nm_core_event_matching_s matching;  
};
void nm_core_monitor_add(nm_core_t p_core, struct nm_core_monitor_s*m);
void nm_core_monitor_remove(nm_core_t p_core, struct nm_core_monitor_s*m);
void nm_core_req_monitor(struct nm_core*p_core, struct nm_req_s*p_req, struct nm_monitor_s monitor);
 
 
#define NM_EVENT_MATCHING_ANY ((struct nm_core_event_matching_s){ .p_gate = NM_ANY_GATE, .tag = NM_CORE_TAG_NONE, .tag_mask = NM_CORE_TAG_NONE })
 
#define NM_MONITOR_NULL ((struct nm_monitor_s){ .p_notifier = NULL, .event_mask = 0, .ref = NULL })
 
#define NM_CORE_MONITOR_NULL ((struct nm_core_monitor_s){ .monitor = NM_MONITOR_NULL, .matching = NM_EVENT_MATCHING_ANY })
 
/* ** Core tasks ******************************************* */
 
enum nm_core_task_kind_e
  {
    NM_CORE_TASK_INVALID            = -1,
    NM_CORE_TASK_NONE               = 0,
    NM_CORE_TASK_NOP                = 1, 
    NM_CORE_TASK_UNPACK_NEXT        = 2, 
    NM_CORE_TASK_COMPLETED_PW       = 3, 
    NM_CORE_TASK_COMPLETED_PREFETCH = 4, 
    NM_CORE_TASK_CANCELLED_PREFETCH = 5, 
    NM_CORE_TASK_PACK_SUBMISSION    = 6, 
    NM_CORE_TASK_RTR_SEND           = 7, 
    NM_CORE_TASK_HANDLER            = 8  
  };
typedef enum nm_core_task_kind_e nm_core_task_kind_t;
 
struct nm_matching_container_s
{
  struct nm_gtag_s*p_gtag;                    
  struct nm_matching_wildcard_s*p_wildcard;   
  struct nm_matching_gsession_s*p_gsession;   
  struct nm_matching_tag_s*p_matching_tag;    
};
#define NM_MATCHING_CONTAINER_NULL ((struct nm_matching_container_s) { NULL })
 
struct nm_core_task_s
{
  enum nm_core_task_kind_e kind;
  union
  {
    struct
    {
      struct nm_req_s*p_unpack;
      struct nm_unexpected_s*p_unexpected;
    } unpack_next;
    struct
    {
      struct nm_pkt_wrap_s*p_pw;
    } completed_pw;
    struct
    {
      struct nm_pkt_wrap_s*p_pw;
    } completed_prefetch;
    struct
    {
      struct nm_pkt_wrap_s*p_pw;
    } cancelled_prefetch;
    struct
    {
      struct nm_req_chunk_s*p_req_chunk;
    } pack_submission;
    struct
    {
      struct nm_pkt_wrap_s*p_pw;
    } rtr_send;
    struct
    {
      void (*p_handler)(void);
    } handler;
  } content;
};
 
void nm_core_task_submit_locked(struct nm_core*p_core, void (*p_handler)(void));
 
void nm_core_task_submit_unlocked(struct nm_core*p_core, void (*p_handler)(void));
 
/* ** Packs/unpacks **************************************** */
 
struct nm_chunk_s
{
  nm_len_t chunk_offset;
  nm_len_t chunk_len;
};
 
PUK_LIST_DECLARE_TYPE(nm_req_chunk);
 
struct nm_req_chunk_s
{
  PUK_LIST_LINK(nm_req_chunk);
  struct nm_core_task_s core_task; 
  struct nm_req_s*p_req;  
  nm_len_t chunk_len;     
  nm_len_t chunk_offset;  
  nm_proto_t proto_flags; 
  struct nm_data_properties_s chunk_props; 
};
 
PUK_LIST_DECLARE_TYPE(nm_req);
 
struct nm_req_s
{
  PUK_LIST_LINK(nm_req);        
  nm_cond_status_t status;      
  struct nm_data_s data;        
  struct nm_monitor_s monitor;  
  nm_req_flag_t flags;          
  nm_gate_t p_gate;             
  nm_core_tag_t tag;            
  nm_seq_t seq;                 
  struct nm_gtag_s*p_gtag;      
  int err;                      
  union
  {
    struct
    {
      nm_len_t len;             
      nm_len_t done;            
      nm_len_t hlen;            
      nm_prio_t priority;       
      uint32_t checksum;        
    } pack;
    struct
    {
      nm_len_t expected_len;  
      nm_len_t cumulated_len; 
      nm_len_t offset;        
      nm_req_seq_t req_seq;   
      nm_core_tag_t tag_mask; 
      struct nm_matching_container_s matching; 
      struct
      {
        int n_partitions;
        struct nm_req_pchunk_s
        {
          struct nm_req_pchunk_s*p_next;
          nm_len_t chunk_offset;
          nm_len_t chunk_len;
        } *p_pchunks;          
      } partition;            
      struct nm_pkt_wrap_s*p_prefetch_pw; 
    } unpack;
  };
  struct nm_req_chunk_s req_chunk; 
};
 
void nm_core_pack_init(struct nm_core*p_core, struct nm_req_s*p_pack);
 
void nm_core_pack_data(nm_core_t p_core, struct nm_req_s*p_pack, const struct nm_data_s*p_data);
 
void nm_core_pack_send(struct nm_core*p_core, struct nm_req_s*p_pack, nm_core_tag_t tag, nm_gate_t p_gate, nm_req_flag_t flags);
 
void nm_core_pack_submit(struct nm_core*p_core, struct nm_req_s*p_pack);
 
void nm_core_pack_set_priority(struct nm_core*p_core, struct nm_req_s*p_pack, nm_prio_t priority);
 
static inline void nm_core_pack_set_hlen(struct nm_core*p_core __attribute__((unused)), struct nm_req_s*p_pack, nm_len_t hlen)
{
  p_pack->pack.hlen = hlen;
}
 
void nm_core_pack_submit_chunks(struct nm_core*p_core, struct nm_req_s*p_pack, int n, const struct nm_chunk_s*p_chunks);
 
void nm_core_unpack_init(struct nm_core*p_core, struct nm_req_s*p_unpack);
 
void nm_core_unpack_offset(struct nm_core*p_core, struct nm_req_s*p_unpack, nm_len_t offset);
 
void nm_core_unpack_data(struct nm_core*p_core, struct nm_req_s*p_unpack, const struct nm_data_s*p_data);
 
void nm_core_unpack_match_recv(struct nm_core*p_core, struct nm_req_s*p_unpack, nm_gate_t p_gate, nm_core_tag_t tag, nm_core_tag_t tag_mask);
 
void nm_core_unpack_match_event(struct nm_core*p_core, struct nm_req_s*p_unpack, const struct nm_core_event_s*p_event);
 
void nm_core_unpack_submit(struct nm_core*p_core, struct nm_req_s*p_unpack, nm_req_flag_t flags);
 
int nm_core_unpack_peek(struct nm_core*p_core, struct nm_req_s*p_unpack, const struct nm_data_s*p_data,
                        nm_len_t peek_offset, nm_len_t peek_len);
 
int nm_core_unpack_iprobe(struct nm_core*p_core, struct nm_req_s*p_unpack);
 
int nm_core_unpack_cancel(struct nm_core*p_core, struct nm_req_s*p_unpack);
 
int nm_core_iprobe(struct nm_core*p_core,
                   nm_gate_t p_gate, nm_core_tag_t tag, nm_core_tag_t tag_mask,
                   nm_gate_t *pp_out_gate, nm_core_tag_t*p_out_tag, nm_len_t*p_out_size);
 
void nm_core_flush(struct nm_core*p_core);
 
 
/* ** Packet injection from outside of nmad core */
 
nm_seq_t nm_core_send_seq_get(struct nm_core*p_core, nm_gate_t p_gate, nm_core_tag_t tag);
 
typedef void(*nm_injector_pull_data_t)(struct nm_req_s*p_req, const struct nm_data_s*p_data, nm_len_t chunk_offset, nm_len_t chunk_len, void*p_ref);
 
void nm_core_inject_chunk(struct nm_core*p_core, nm_gate_t p_gate, nm_core_tag_t tag, nm_seq_t seq,
                          nm_len_t chunk_offset, nm_len_t chunk_len, int is_last_chunk,
                          nm_injector_pull_data_t p_pull_data, void*p_ref);
 
void nm_core_inject_complete(struct nm_core*p_core, struct nm_req_s*p_req, nm_len_t chunk_offset, nm_len_t chunk_len);
 
void nm_core_inject_finalize(struct nm_core*p_core, struct nm_req_s*p_req);
 
void nm_core_inject_complete_finalize(struct nm_core*p_core, struct nm_req_s*p_req, nm_len_t chunk_offset, nm_len_t chunk_len);
 
/* ** partitioned unpack */
 
void nm_core_unpack_partition_set(struct nm_req_s*p_unpack, int n_partitions);
 
void nm_core_unpack_partition_free(struct nm_req_s*p_unpack);
 
int nm_core_unpack_partition_test(struct nm_req_s*p_unpack, int partition);
 
/* ** synchronization primitives *************************** */
 
static inline void nm_cond_init(nm_cond_status_t*p_cond, nm_status_t bitmask);
 
static inline void nm_cond_destroy(nm_cond_status_t*p_cond);
 
static inline nm_status_t nm_cond_test(const nm_cond_status_t*p_cond, nm_status_t bitmask);
 
static inline nm_status_t nm_cond_test_locked(const nm_cond_status_t*p_cond, nm_status_t bitmask);
 
static inline void nm_cond_add(nm_cond_status_t*p_cond, nm_status_t bitmask);
 
static inline void nm_cond_wait(nm_cond_status_t*p_cond, nm_status_t bitmask, nm_core_t p_core);
 
static inline void nm_cond_signal(nm_cond_status_t*p_cond, nm_status_t bitmask);
 
static inline void nm_cond_wait_all(void**pp_conds, int n, uintptr_t offset, nm_status_t bitmask, nm_core_t p_core);
 
#if defined(PIOMAN)
 
static inline void nm_cond_init(nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  piom_cond_init(p_cond, bitmask);
}
static inline void nm_cond_destroy(nm_cond_status_t*p_cond)
{
  piom_cond_destroy(p_cond);
}
static inline nm_status_t nm_cond_test(const nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  return piom_cond_test(p_cond, bitmask);
}
static inline nm_status_t nm_cond_test_locked(const nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  return piom_cond_test_locked((nm_cond_status_t*)p_cond, bitmask);
}
static inline void nm_cond_add(nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  piom_cond_add(p_cond, bitmask);
}
static inline void nm_cond_mask(nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  piom_cond_mask(p_cond, bitmask);
}
static inline void nm_cond_wait(nm_cond_status_t*p_cond, nm_status_t bitmask, nm_core_t p_core __attribute__((unused)))
{
  piom_cond_wait(p_cond, bitmask);
}
static inline void nm_cond_signal(nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  piom_cond_signal(p_cond, bitmask);
}
static inline void nm_cond_wait_all(void**pp_conds, int n, uintptr_t offset, nm_status_t bitmask, nm_core_t p_core  __attribute__((unused)))
{
  piom_cond_wait_all(pp_conds, n, offset, bitmask);
}
#else /* PIOMAN */
static inline void nm_cond_init(nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  *p_cond = bitmask;
}
static inline void nm_cond_destroy(nm_cond_status_t*p_cond __attribute__((unused)))
{
}
static inline nm_status_t nm_cond_test(const nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  return ((*p_cond) & bitmask);
}
static inline nm_status_t nm_cond_test_locked(const nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  return nm_cond_test(p_cond, bitmask);
}
static inline void nm_cond_add(nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  *p_cond |= bitmask;
}
static inline void nm_cond_mask(nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  *p_cond &= ~bitmask;
}
static inline void nm_cond_signal(nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  *p_cond |= bitmask;
}
static inline void nm_cond_wait(nm_cond_status_t*p_cond, nm_status_t bitmask, nm_core_t p_core)
{
  while(!nm_cond_test(p_cond, bitmask))
    {
      nm_schedule(p_core);
    }
}
static inline void nm_cond_wait_all(void**pp_conds, int n, uintptr_t offset, nm_status_t bitmask, nm_core_t p_core)
{
  int i;
  for(i = 0; i < n; i++)
    {
      if(pp_conds[i] != NULL)
        {
          nm_cond_status_t*p_cond = (nm_cond_status_t*)((uintptr_t)pp_conds[i] + offset);
          nm_cond_wait(p_cond, bitmask, p_core);
        }
    }
}
#endif /* PIOMAN */
 
/* ** convenient frontends to deal with status in requests */
 
static inline void nm_status_init(struct nm_req_s*p_req, nm_status_t bitmask)
{
  nm_cond_init(&p_req->status, bitmask);
}
static inline void nm_status_destroy(struct nm_req_s*p_req)
{
  nm_cond_destroy(&p_req->status);
}
static inline nm_status_t nm_status_test(const struct nm_req_s*p_req, nm_status_t bitmask)
{
  if(bitmask & NM_STATUS_FINALIZED) /* status FINALIZED needs strong consistency to avoid use after free */
    return nm_cond_test_locked(&p_req->status, bitmask);
  else
    return nm_cond_test(&p_req->status, bitmask);
}
static inline void nm_status_add(struct nm_req_s*p_req, nm_status_t bitmask)
{
  nm_cond_add(&p_req->status, bitmask);
}
static inline void nm_status_unset(struct nm_req_s*p_req, nm_status_t bitmask)
{
  nm_cond_mask(&p_req->status, bitmask);
}
static inline void nm_status_wait(struct nm_req_s*p_req, nm_status_t bitmask, nm_core_t p_core)
{
  nm_cond_wait(&p_req->status, bitmask, p_core);
  assert(nm_status_test(p_req, bitmask) != 0);
}
static inline void nm_status_signal(struct nm_req_s*p_req, nm_status_t bitmask)
{
  nm_cond_signal(&p_req->status, bitmask);
}
static inline void nm_status_wait_all(void**pp_reqs, int n, uintptr_t offset,
                                      nm_status_t bitmask, nm_core_t p_core)
{
  const struct nm_req_s*p_req = NULL;
  const uintptr_t status_offset = (uintptr_t)&p_req->status - (uintptr_t)p_req; /* offset of 'status' in nm_req_s */
  nm_cond_wait_all(pp_reqs, n, offset + status_offset, bitmask, p_core);
}
static inline void nm_status_assert(struct nm_req_s*p_req __attribute__((unused)), nm_status_t value __attribute__((unused)))
{
  assert(nm_status_test(p_req, NM_STATUS_MASK_FULL) == value);
}
 
static inline void nm_status_spinwait(struct nm_req_s*p_req, nm_status_t status)
{
  while(!nm_status_test(p_req, status))
    { /* bust wait*/ }
}
static inline int nm_status_test_allbits(struct nm_req_s*p_req, nm_status_t bitmask)
{
  return (nm_status_test(p_req, bitmask) == bitmask);
}
 
/* ** frontends for atomic ops ***************************** */
 
static inline void nm_mem_fence_always(void)
{
  __sync_synchronize();
}
 
static inline void nm_mem_fence(void)
{
#if defined(PIOMAN_MULTITHREAD)
  __sync_synchronize();
#else
  nm_core_t p_core = nm_core_get_singleton();
  if(nm_core_get_thread_level(p_core) >= NM_THREAD_SERIALIZED)
    {
      __sync_synchronize();
    }
#endif /* PIOMAN_MULTITHREAD */
}
 
static inline int nm_atomic_inc(int*v)
{
#if defined(PIOMAN_MULTITHREAD)
  return __sync_fetch_and_add(v, 1);
#else
  nm_core_t p_core = nm_core_get_singleton();
  if(nm_core_get_thread_level(p_core) >= NM_THREAD_SERIALIZED)
    {
      return __sync_fetch_and_add(v, 1);
    }
  else
    {
      return (*v)++;
    }
#endif /* PIOMAN_MULTITHREAD */
}
 
static inline int nm_atomic_alway_inc(int*v)
{
  return __sync_fetch_and_add(v, 1);
}
 
static inline int nm_atomic_dec(int*v)
{
#if defined(PIOMAN_MULTITHREAD)
  return __sync_sub_and_fetch(v, 1);
#else
  nm_core_t p_core = nm_core_get_singleton();
  if(nm_core_get_thread_level(p_core) >= NM_THREAD_SERIALIZED)
    {
      return __sync_sub_and_fetch(v, 1);
    }
  else
    {
      return --(*v);
    }
#endif /* PIOMAN_MULTITHREAD */
}
 
static inline int nm_atomic_always_dec(int*v)
{
  return __sync_sub_and_fetch(v, 1);
}
 
static inline void nm_atomic_add(int*v, int v2)
{
#if defined(PIOMAN_MULTITHREAD)
  __sync_fetch_and_add(v, v2);
#else
  nm_core_t p_core = nm_core_get_singleton();
  if(nm_core_get_thread_level(p_core) >= NM_THREAD_SERIALIZED)
    {
      __sync_fetch_and_add(v, v2);
    }
  else
    {
      (*v) += v2;
    }
#endif /* PIOMAN_MULTITHREAD */
}
 
static inline void nm_atomic_always_add(int*v, int v2)
{
  __sync_fetch_and_add(v, v2);
}
 
static inline int nm_atomic_compare_and_swap(int*v, int oldval, int newval)
{
#if defined(PIOMAN_MULTITHREAD)
  return __sync_bool_compare_and_swap(v, oldval, newval);
#else
  nm_core_t p_core = nm_core_get_singleton();
  if(nm_core_get_thread_level(p_core) >= NM_THREAD_SERIALIZED)
    {
      return __sync_bool_compare_and_swap(v, oldval, newval);
    }
  else
    {
      if(*v == oldval)
        {
          *v = newval;
          return 1;
        }
      else
        {
          return 0;
        }
    }
#endif /* PIOMAN_MULTITHREAD */
}
 
static inline int nm_atomic_always_compare_and_swap(int*v, int oldval, int newval)
{
  return __sync_bool_compare_and_swap(v, oldval, newval);
}
 
/* ** frontend for generic locking ************************* */
 
#ifdef PIOMAN
typedef piom_spinlock_t nm_spinlock_t;
#else /* PIOMAN */
struct nm_spinlock_s
{
#ifdef NMAD_DEBUG
  int lock;
  pthread_t last_tid;
#endif /* NMAD_DEBUG */
};
typedef struct nm_spinlock_s nm_spinlock_t;
#endif /* PIOMAN */
 
static inline void nm_spin_init(nm_spinlock_t*p_spin);
 
static inline void nm_spin_destroy(nm_spinlock_t*p_spin);
 
static inline void nm_spin_lock(nm_spinlock_t*p_spin);
 
static inline void nm_spin_unlock(nm_spinlock_t*p_spin);
 
static inline int nm_spin_trylock(nm_spinlock_t*p_spin);
 
static inline void nm_spin_assert_locked(nm_spinlock_t*p_spin);
 
static inline void nm_spin_assert_notlocked(nm_spinlock_t*p_spin);
 
 
static inline void nm_spin_check_nothread(nm_spinlock_t*p_spin __attribute__((unused)))
{
#if defined(NMAD_DEBUG) && !defined(PIOMAN)
  nm_core_t p_core = nm_core_get_singleton();
  assert(nm_core_get_thread_level(p_core) <= NM_THREAD_SERIALIZED);
  if(p_spin->last_tid == (pthread_t)0)
    {
      p_spin->last_tid = pthread_self();
      __sync_synchronize();
    }
  else
    {
      if(p_spin->last_tid != pthread_self())
        {
          NM_FATAL("detected calls from multiple threads in non-threaded mode. Please use pioman-enabled build for multi-threaded use or give thread level using nm_core_set_thread_level(NM_THREAD_SERIALIZED) for serialized thread level.");
        }
    }
#endif /* NMAD_DEBUG && !PIOMAN */
}
 
static inline void nm_spin_clear_nothread(nm_spinlock_t*p_spin __attribute__((unused)))
{
#if defined(NMAD_DEBUG) && !defined(PIOMAN)
  nm_core_t p_core = nm_core_get_singleton();
  assert(nm_core_get_thread_level(p_core) <= NM_THREAD_SERIALIZED);
  if(p_spin->last_tid == 0)
    {
      NM_FATAL("unlocking while no thread is holding the lock.");
    }
  else if(p_spin->last_tid != pthread_self())
    {
      NM_WARN("unlocking from another thread than where lock was acquired.\n");
    }
  if(nm_core_get_thread_level(p_core) == NM_THREAD_SERIALIZED)
    {
      p_spin->last_tid = (pthread_t)0;
    }
#endif /* NMAD_DEBUG && !PIOMAN */
}
 
static inline void nm_spin_init(nm_spinlock_t*p_spin __attribute__((unused)))
{
#ifdef PIOMAN
  piom_spin_init(p_spin);
#else /* PIOMAN */
#ifdef NMAD_DEBUG
  p_spin->lock = 0;
  p_spin->last_tid = 0;
#endif /* NMAD_DEBUG */
#endif /* PIOMAN */
}
 
static inline void nm_spin_destroy(nm_spinlock_t*p_spin __attribute__((unused)))
{
#ifdef PIOMAN
  piom_spin_destroy(p_spin);
#else /* PIOMAN */
#ifdef NMAD_DEBUG
  assert(p_spin->lock == 0);
#endif /* NMAD_DEBUG */
#endif /* PIOMAN */
}
 
static inline void nm_spin_lock(nm_spinlock_t*p_spin __attribute__((unused)))
{
#ifdef PIOMAN
  piom_spin_lock(p_spin);
#else /* PIOMAN */
#ifdef NMAD_DEBUG
  __sync_synchronize();
  if(p_spin->lock != 0)
    {
      NM_FATAL("spinlock is not free in nm_spin_lock(); detected concurrent access from thread = %p. Suspecting multi-threaded use by the application while library is initialized in non-threaded mode.\n",
               (void*)p_spin->last_tid);
    }
  p_spin->lock = 1;
#endif /* NMAD_DEBUG */
  nm_spin_check_nothread(p_spin);
  nm_core_t p_core = nm_core_get_singleton();
  if(nm_core_get_thread_level(p_core) >= NM_THREAD_SERIALIZED)
    {
      __sync_synchronize();
    }
#endif /* PIOMAN */
}
 
static inline void nm_spin_unlock(nm_spinlock_t*p_spin __attribute__((unused)))
{
#ifdef PIOMAN
  piom_spin_unlock(p_spin);
#else /* PIOMAN */
  nm_spin_clear_nothread(p_spin);
#ifdef NMAD_DEBUG
  __sync_synchronize();
  assert(p_spin->lock == 1);
  p_spin->lock = 0;
#endif /* NMAD_DEBUG */
  nm_core_t p_core = nm_core_get_singleton();
  if(nm_core_get_thread_level(p_core) >= NM_THREAD_SERIALIZED)
    {
      __sync_synchronize();
    }
#endif /* PIOMAN */
}
 
static inline int nm_spin_trylock(nm_spinlock_t*p_spin __attribute__((unused)))
{
#ifdef PIOMAN
  return piom_spin_trylock(p_spin);
#else /* PIOMAN */
  int rc = 1;
#ifdef NMAD_DEBUG
  __sync_synchronize();
  if(p_spin->lock)
    {
      assert(p_spin->lock == 1);
      rc = 0;
    }
  else
    {
      rc = 1;
      p_spin->lock = 1;
      nm_spin_check_nothread(p_spin);
    }
#endif /* NMAD_DEBUG */
  nm_core_t p_core = nm_core_get_singleton();
  if(nm_core_get_thread_level(p_core) >= NM_THREAD_SERIALIZED)
    {
      __sync_synchronize();
    }
  return rc;
#endif /* PIOMAN */
}
 
static inline void nm_spin_assert_locked(nm_spinlock_t*p_spin __attribute__((unused)))
{
#ifdef PIOMAN
  piom_spin_assert_locked(p_spin);
#else /* PIOMAN */
#ifdef NMAD_DEBUG
  assert(p_spin->lock == 1);
  assert(p_spin->last_tid == pthread_self());
#endif /* NMAD_DEBUG */
#endif /* PIOMAN */
}
 
static inline void nm_spin_assert_notlocked(nm_spinlock_t*p_spin __attribute__((unused)))
{
#ifdef PIOMAN
  piom_spin_assert_notlocked(p_spin);
#else /* PIOMAN */
#ifdef NMAD_DEBUG
  assert(p_spin->lock == 0);
#endif /* NMAD_DEBUG */
#endif /* PIOMAN */
}
 
/* ** refcount ********************************************* */
 
PUK_LIST_TYPE(nm_refcount_holder,
              const void*p_holder; 
              const char*func;     
              const char*file;     
              int line;            
              );
 
struct nm_refcount_s
{
  int refcount;                    
#ifdef NMAD_DEBUG
  char*object_id;                  
  nm_spinlock_t lock;              
  struct nm_refcount_holder_list_s holders; 
#endif /* NMAD_DEBUG */
};
 
static inline void nm_refcount_dump(struct nm_refcount_s*p_refcount __attribute__((unused)))
{
#ifdef NMAD_DEBUG
  nm_refcount_holder_itor_t i;
  puk_list_foreach(nm_refcount_holder, i, &p_refcount->holders)
    {
      NM_WARN("pending ref- object = %s; p_holder = %p; %s:%d %s()\n",
              p_refcount->object_id, i->p_holder, i->file, i->line, i->func);
    }
#endif /* NMAD_DEBUG */
}
 
static inline void nm_refcount_destroy(struct nm_refcount_s*p_refcount __attribute__((unused)))
{
#ifdef NMAD_DEBUG
  nm_refcount_dump(p_refcount);
  nm_spin_destroy(&p_refcount->lock);
  padico_free(p_refcount->object_id);
  nm_refcount_holder_list_destroy(&p_refcount->holders);
#endif /* NMAD_DEBUG */
}
 
static inline int nm_refcount_get(struct nm_refcount_s*p_refcount)
{
  return p_refcount->refcount;
}
 
#define nm_refcount_inc(REFCOUNT, HOLDER)   \
  nm_refcount_inc_internal(REFCOUNT, HOLDER, __FUNCTION__, __FILE__, __LINE__)
 
#ifdef NMAD_DEBUG
static inline void nm_refcount_add_holder(struct nm_refcount_s*p_refcount, const void*p_holder,
                                          const char*func, const char*file, const int line)
{
  struct nm_refcount_holder_s*h = nm_refcount_holder_new();
  h->p_holder = p_holder;
  h->func = func;
  h->file = file;
  h->line = line;
  nm_refcount_holder_list_push_back(&p_refcount->holders, h);
}
 
static inline int nm_refcount_count_holder(struct nm_refcount_s*p_refcount, const void*p_holder)
{
  int count = 0;
  nm_refcount_holder_itor_t i;
  puk_list_foreach(nm_refcount_holder, i, &p_refcount->holders)
    {
      if(i->p_holder == p_holder)
        count++;
    }
  return count;
}
#endif /* NMAD_DEBUG */
 
static inline void nm_refcount_inc_internal(struct nm_refcount_s*p_refcount, const void*p_holder __attribute__((unused)),
                                            const char*func __attribute__((unused)),
                                            const char*file __attribute__((unused)),
                                            const int line __attribute__((unused)))
{
  assert(p_refcount->refcount >= 0);
  nm_atomic_inc(&p_refcount->refcount);
#ifdef NMAD_DEBUG
  if(p_holder != NULL)
    {
      nm_spin_lock(&p_refcount->lock);
#if 0
      if(nm_refcount_count_holder(p_refcount, p_holder) > 0)
        {
          nm_refcount_holder_itor_t i;
          PUK_LIST_FIND(nm_refcount_holder, i, &p_refcount->holders, (i->p_holder == p_holder));
          NM_WARN("p_holder = %p already holding a ref to p_refcount = %p (%s)\n"
                  " new holder: %s:%d %s()\n"
                  " previous holder: %s:%d %s()\n",
                  p_holder, p_refcount, p_refcount->object_id,
                  file, line, func, i->file, i->line, i->func);
        }
#endif /* 0 */
      nm_refcount_add_holder(p_refcount, p_holder, func, file, line);
      nm_spin_unlock(&p_refcount->lock);
    }
#endif /* NMAD_DEBUG */
}
 
static inline int nm_refcount_dec(struct nm_refcount_s*p_refcount, const void*p_holder __attribute__((unused)))
{
  const int nb_ref = nm_atomic_dec(&p_refcount->refcount);
  assert(nb_ref >= 0);
#ifdef NMAD_DEBUG
  if(p_holder != NULL)
    {
      nm_spin_lock(&p_refcount->lock);
      const int count = nm_refcount_count_holder(p_refcount, p_holder);
      nm_refcount_holder_itor_t i;
      PUK_LIST_FIND(nm_refcount_holder, i, &p_refcount->holders, (i->p_holder == p_holder));
      if(i == NULL)
        {
          NM_FATAL("p_holder = %p not holding ref to p_refcount = %p (%s); cannot release\n",
                   p_holder, p_refcount, p_refcount->object_id);
        }
      else
        {
          nm_refcount_holder_list_remove(&p_refcount->holders, i);
          nm_refcount_holder_delete(i);
        }
      assert(nm_refcount_count_holder(p_refcount, p_holder) == count - 1);
      nm_spin_unlock(&p_refcount->lock);
    }
#endif /* NMAD_DEBUG */
  return nb_ref;
}
 
static inline void nm_refcount_init(struct nm_refcount_s*p_refcount, char*p_object_id, const void*p_init_holder)
{
  p_refcount->refcount = 1;
#ifdef NMAD_DEBUG
  p_refcount->object_id = p_object_id;
  nm_spin_init(&p_refcount->lock);
  nm_refcount_holder_list_init(&p_refcount->holders);
  if(p_init_holder != NULL)
    {
      nm_refcount_add_holder(p_refcount, p_init_holder, __FUNCTION__, __FILE__, __LINE__);
    }
#else /* NMAD_DEBUG */
  if(p_object_id != NULL)
    padico_free(p_object_id); /* when not in debug, we won't use the object_id at all */
#endif /* NMAD_DEBUG */
}
 
 
#endif /* NM_CORE_INTERFACE_H */