nm_core_interface.h

Go to the documentation of this file.

/*
 * NewMadeleine
 * Copyright (C) 2006-2025 (see AUTHORS file)
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 */
 
 
#ifndef NM_CORE_INTERFACE_H
#define NM_CORE_INTERFACE_H
 
#include <nm_public.h>
#include <nm_log.h>
#include <Padico/Puk.h>
#include <sys/uio.h>
 
#ifdef PIOMAN
#include <pioman.h>
#else
#include <pthread.h>
#endif
 
/* ** Core init ******************************************** */
 
typedef struct nm_core*nm_core_t;
 
puk_component_t nm_core_component_load(const char*entity, const char*name);
 
int nm_core_init(nm_core_t *pp_core);
 
int nm_core_set_strategy(nm_core_t p_core, puk_component_t strategy);
 
int nm_core_exit(nm_core_t p_core);
 
void nm_core_schedopt_disable(nm_core_t p_core);
 
extern struct nm_core_internal_s
{
  nm_core_t p_core_singleton;
} nm_core_internal;
 
__attribute__((pure))
static inline nm_core_t nm_core_get_singleton(void)
{
  return nm_core_internal.p_core_singleton;
}
 
void nm_trace_add_synchro_point(void);
 
 
/* ** Drivers ********************************************** */
 
typedef struct nm_drv_s*nm_drv_t;
 
PUK_VECT_TYPE(nm_drv, nm_drv_t);
 
int nm_core_driver_load_init(nm_core_t p_core, puk_component_t driver, nm_trk_kind_t kind,
                             nm_drv_t *pp_drv, const char**p_url);
 
 
/* ** Gates ************************************************ */
 
nm_gate_t nm_core_gate_new(nm_core_t p_core, nm_drv_vect_t*p_drvs);
void nm_core_gate_connect_async(nm_core_t p_core, nm_gate_t gate, nm_drv_t p_drv, nm_trk_id_t trk_id, const char*url);
void nm_core_gate_connect_wait(nm_core_t p_core, struct nm_trk_s*p_trk);
 
 
/* ** Threads ********************************************** */
 
enum nm_thread_level_e
  {
    NM_THREAD_SINGLE     = 0,
    NM_THREAD_FUNNELED   = 1,
    NM_THREAD_SERIALIZED = 2,
    NM_THREAD_MULTIPLE   = 3
  };
typedef enum nm_thread_level_e nm_thread_level_t;
 
nm_thread_level_t nm_core_get_thread_level(nm_core_t);
 
void nm_core_set_thread_level(nm_thread_level_t);
 
/* ** Progression ****************************************** */
 
int nm_schedule(nm_core_t p_core);
 
/* ** Status *********************************************** */
 
#ifdef PIOMAN
typedef piom_cond_value_t nm_status_t;
typedef piom_cond_t nm_cond_status_t;
#else /* PIOMAN */
typedef uint32_t nm_status_t;
typedef nm_status_t nm_cond_status_t;
#endif /* PIOMAN */
 
/* ** status and flags, used in pack/unpack requests and events */
 
#define NM_STATUS_NONE                     ((nm_status_t)0x00000000)
#define NM_STATUS_PACK_INIT                ((nm_status_t)0x00000001)
#define NM_STATUS_UNPACK_INIT              ((nm_status_t)0x00000002)
#define NM_STATUS_PACK_COMPLETED           ((nm_status_t)0x00000004)
#define NM_STATUS_UNPACK_COMPLETED         ((nm_status_t)0x00000008)
#define NM_STATUS_UNEXPECTED               ((nm_status_t)0x00000010)
#define NM_STATUS_UNPACK_CANCELLED         ((nm_status_t)0x00000020)
#define NM_STATUS_PACK_POSTED              ((nm_status_t)0x00000040)
#define NM_STATUS_UNPACK_POSTED            ((nm_status_t)0x00000080)
#define NM_STATUS_ACK_RECEIVED             ((nm_status_t)0x00000100)
#define NM_STATUS_UNPACK_DATA0             ((nm_status_t)0x00000200)
#define NM_STATUS_UNPACK_DATA_SIZE         ((nm_status_t)0x00000400)
#define NM_STATUS_FINALIZED                ((nm_status_t)0x00000800)
#define NM_STATUS_ERROR                    ((nm_status_t)0x00001000)
#define NM_STATUS_PACK_MSG_SIZE            ((nm_status_t)0x00002000)
#define NM_STATUS_UNPACK_PREFETCHED        ((nm_status_t)0x00004000)
 
#define NM_STATUS_MASK_FULL                ((nm_status_t)-1)
 
 
typedef uint32_t nm_req_flag_t;
 
#define NM_REQ_FLAG_NONE                   ((nm_req_flag_t)0x00000000)
#define NM_REQ_FLAG_PACK_SYNCHRONOUS       ((nm_req_flag_t)0x00001000)
#define NM_REQ_FLAG_PACK                   ((nm_req_flag_t)0x00002000)
#define NM_REQ_FLAG_UNPACK                 ((nm_req_flag_t)0x00004000)
#define NM_REQ_FLAG_UNPACK_DATA_INFO       ((nm_req_flag_t)0x00008000)
#define NM_REQ_FLAG_UNPACK_MATCHING_INFO   ((nm_req_flag_t)0x00010000)
#define NM_REQ_FLAG_UNPACK_PREFETCHING     ((nm_req_flag_t)0x00020000)
#define NM_REQ_FLAG_MATCHING_WILDCARD      ((nm_req_flag_t)0x00100000)
#define NM_REQ_FLAG_MATCHING_GATE          ((nm_req_flag_t)0x00200000)
#define NM_REQ_FLAG_MATCHING_TAG           ((nm_req_flag_t)0x00400000)
#define NM_REQ_FLAG_MATCHING_FULL          ((nm_req_flag_t)0x00800000)
#define NM_REQ_FLAG_FINALIZE_LATER         ((nm_req_flag_t)0x01000000)
#define NM_REQ_FLAG_UNPACK_PARTITIONED     ((nm_req_flag_t)0x02000000)
#define NM_REQ_FLAG_PACK_PARTITIONED       ((nm_req_flag_t)0x04000000)
 
typedef uint32_t nm_req_chunk_flag_t;
 
#define NM_REQ_CHUNK_FLAG_NONE             ((nm_req_chunk_flag_t)0x00000000)
#define NM_REQ_CHUNK_FLAG_SHORT            ((nm_req_chunk_flag_t)0x00020000)
#define NM_REQ_CHUNK_FLAG_USE_COPY         ((nm_req_chunk_flag_t)0x00080000)
#define NM_REQ_CHUNK_FLAG_DATA_ITERATOR    ((nm_req_chunk_flag_t)0x00100000)
 
/* ** tags ************************************************* */
 
typedef uint32_t nm_session_hash_t;
 
#define NM_CORE_TAG_HASH_FULL ((nm_session_hash_t)0xFFFFFFFF)
 
struct nm_core_tag_s
{
  nm_tag_t tag;               
  nm_session_hash_t hashcode; 
} __attribute__((packed));
 
typedef struct nm_core_tag_s nm_core_tag_t;
 
#define NM_CORE_TAG_MASK_FULL ((nm_core_tag_t){ .tag = NM_TAG_MASK_FULL, .hashcode = NM_CORE_TAG_HASH_FULL })
#define NM_CORE_TAG_NONE      ((nm_core_tag_t){ .tag = 0, .hashcode = 0x0 })
 
static inline nm_core_tag_t nm_core_tag_build(nm_session_hash_t hashcode, nm_tag_t tag)
{
  nm_core_tag_t core_tag;
  core_tag.tag = tag;
  core_tag.hashcode = hashcode;
  return core_tag;
}
static inline nm_tag_t nm_core_tag_get_tag(nm_core_tag_t core_tag)
{
  return core_tag.tag;
}
static inline nm_session_hash_t nm_core_tag_get_hashcode(nm_core_tag_t core_tag)
{
  return core_tag.hashcode;
}
 
/* ** Event notification *********************************** */
 
struct nm_core_event_s
{
  nm_status_t status; 
  nm_gate_t p_gate;
  nm_core_tag_t tag;
  nm_seq_t seq;
  nm_len_t len;
  struct nm_req_s*p_req; 
};
 
typedef void (*nm_core_event_notifier_t)(const struct nm_core_event_s*const event, void*ref);
 
struct nm_core_event_matching_s
{
  nm_gate_t p_gate;       
  nm_core_tag_t tag;      
  nm_core_tag_t tag_mask; 
};
 
struct nm_monitor_s
{
  nm_core_event_notifier_t p_notifier; 
  nm_status_t event_mask;              
  void*ref;                            
};
 
struct nm_core_monitor_s
{
  struct nm_monitor_s monitor;               
  struct nm_core_event_matching_s matching;  
};
void nm_core_monitor_add(nm_core_t p_core, struct nm_core_monitor_s*m);
void nm_core_monitor_remove(nm_core_t p_core, struct nm_core_monitor_s*m);
void nm_core_req_monitor(struct nm_core*p_core, struct nm_req_s*p_req, struct nm_monitor_s monitor);
 
 
#define NM_EVENT_MATCHING_ANY ((struct nm_core_event_matching_s){ .p_gate = NM_ANY_GATE, .tag = NM_CORE_TAG_NONE, .tag_mask = NM_CORE_TAG_NONE })
 
#define NM_MONITOR_NULL ((struct nm_monitor_s){ .p_notifier = NULL, .event_mask = 0, .ref = NULL })
 
#define NM_CORE_MONITOR_NULL ((struct nm_core_monitor_s){ .monitor = NM_MONITOR_NULL, .matching = NM_EVENT_MATCHING_ANY })
 
/* ** Core tasks ******************************************* */
 
enum nm_core_task_kind_e
  {
   NM_CORE_TASK_NONE               = 0,
   NM_CORE_TASK_UNPACK_NEXT        = 1, 
   NM_CORE_TASK_COMPLETED_PW       = 2, 
   NM_CORE_TASK_COMPLETED_PREFETCH = 3, 
   NM_CORE_TASK_PACK_SUBMISSION    = 4, 
   NM_CORE_TASK_RTR_SEND           = 5, 
   NM_CORE_TASK_HANDLER            = 6  
  };
typedef enum nm_core_task_kind_e nm_core_task_kind_t;
 
struct nm_matching_container_s
{
  struct nm_gtag_s*p_gtag;                    
  struct nm_matching_wildcard_s*p_wildcard;   
  struct nm_matching_gsession_s*p_gsession;   
  struct nm_matching_tag_s*p_matching_tag;    
};
#define NM_MATCHING_CONTAINER_NULL ((struct nm_matching_container_s) { NULL })
 
struct nm_core_task_s
{
  enum nm_core_task_kind_e kind;
  union
  {
    struct
    {
      struct nm_req_s*p_unpack;
      struct nm_unexpected_s*p_unexpected;
    } unpack_next;
    struct
    {
      struct nm_pkt_wrap_s*p_pw;
    } completed_pw;
    struct
    {
      struct nm_pkt_wrap_s*p_pw;
    } completed_prefetch;
    struct
    {
      struct nm_req_chunk_s*p_req_chunk;
    } pack_submission;
    struct
    {
      struct nm_pkt_wrap_s*p_pw;
    } rtr_send;
    struct
    {
      void (*p_handler)(void);
    } handler;
  } content;
};
 
void nm_core_task_submit_locked(struct nm_core*p_core, void (*p_handler)(void));
 
void nm_core_task_submit_unlocked(struct nm_core*p_core, void (*p_handler)(void));
 
/* ** Packs/unpacks **************************************** */
 
struct nm_chunk_s
{
  nm_len_t chunk_offset;
  nm_len_t chunk_len;
};
 
PUK_LIST_DECLARE_TYPE(nm_req_chunk);
 
struct nm_req_chunk_s
{
  PUK_LIST_LINK(nm_req_chunk);
  struct nm_core_task_s core_task; 
  struct nm_req_s*p_req;  
  nm_len_t chunk_len;     
  nm_len_t chunk_offset;  
  nm_proto_t proto_flags; 
  struct nm_data_properties_s chunk_props; 
};
 
PUK_LIST_DECLARE_TYPE(nm_req);
 
struct nm_req_s
{
  PUK_LIST_LINK(nm_req);        
  nm_cond_status_t status;      
  struct nm_data_s data;        
  struct nm_monitor_s monitor;  
  nm_req_flag_t flags;          
  nm_gate_t p_gate;             
  nm_core_tag_t tag;            
  nm_seq_t seq;                 
  struct nm_gtag_s*p_gtag;      
  int err;                      
  union
  {
    struct
    {
      nm_len_t len;             
      nm_len_t done;            
      nm_len_t hlen;            
      nm_prio_t priority;       
      uint32_t checksum;        
    } pack;
    struct
    {
      nm_len_t expected_len;  
      nm_len_t cumulated_len; 
      nm_len_t offset;        
      nm_req_seq_t req_seq;   
      nm_core_tag_t tag_mask; 
      struct nm_matching_container_s matching; 
      struct
      {
        int n_partitions;
        struct nm_req_pchunk_s
        {
          struct nm_req_pchunk_s*p_next;
          nm_len_t chunk_offset;
          nm_len_t chunk_len;
        } *p_pchunks;          
      } partition;            
      struct nm_pkt_wrap_s*p_prefetch_pw; 
    } unpack;
  };
  struct nm_req_chunk_s req_chunk; 
};
 
void nm_core_pack_init(struct nm_core*p_core, struct nm_req_s*p_pack);
 
void nm_core_pack_data(nm_core_t p_core, struct nm_req_s*p_pack, const struct nm_data_s*p_data);
 
void nm_core_pack_send(struct nm_core*p_core, struct nm_req_s*p_pack, nm_core_tag_t tag, nm_gate_t p_gate, nm_req_flag_t flags);
 
void nm_core_pack_submit(struct nm_core*p_core, struct nm_req_s*p_pack);
 
void nm_core_pack_set_priority(struct nm_core*p_core, struct nm_req_s*p_pack, nm_prio_t priority);
 
static inline void nm_core_pack_set_hlen(struct nm_core*p_core __attribute__((unused)), struct nm_req_s*p_pack, nm_len_t hlen)
{
  p_pack->pack.hlen = hlen;
}
 
void nm_core_pack_submit_chunks(struct nm_core*p_core, struct nm_req_s*p_pack, int n, const struct nm_chunk_s*p_chunks);
 
void nm_core_unpack_init(struct nm_core*p_core, struct nm_req_s*p_unpack);
 
void nm_core_unpack_offset(struct nm_core*p_core, struct nm_req_s*p_unpack, nm_len_t offset);
 
void nm_core_unpack_data(struct nm_core*p_core, struct nm_req_s*p_unpack, const struct nm_data_s*p_data);
 
void nm_core_unpack_match_recv(struct nm_core*p_core, struct nm_req_s*p_unpack, nm_gate_t p_gate, nm_core_tag_t tag, nm_core_tag_t tag_mask);
 
void nm_core_unpack_match_event(struct nm_core*p_core, struct nm_req_s*p_unpack, const struct nm_core_event_s*p_event);
 
void nm_core_unpack_submit(struct nm_core*p_core, struct nm_req_s*p_unpack, nm_req_flag_t flags);
 
int nm_core_unpack_peek(struct nm_core*p_core, struct nm_req_s*p_unpack, const struct nm_data_s*p_data,
                        nm_len_t peek_offset, nm_len_t peek_len);
 
int nm_core_unpack_iprobe(struct nm_core*p_core, struct nm_req_s*p_unpack);
 
int nm_core_unpack_cancel(struct nm_core*p_core, struct nm_req_s*p_unpack);
 
int nm_core_iprobe(struct nm_core*p_core,
                   nm_gate_t p_gate, nm_core_tag_t tag, nm_core_tag_t tag_mask,
                   nm_gate_t *pp_out_gate, nm_core_tag_t*p_out_tag, nm_len_t*p_out_size);
 
void nm_core_flush(struct nm_core*p_core);
 
 
/* ** Packet injection from outside of nmad core */
 
nm_seq_t nm_core_send_seq_get(struct nm_core*p_core, nm_gate_t p_gate, nm_core_tag_t tag);
 
typedef void(*nm_injector_pull_data_t)(struct nm_req_s*p_req, const struct nm_data_s*p_data, nm_len_t chunk_offset, nm_len_t chunk_len, void*p_ref);
 
void nm_core_inject_chunk(struct nm_core*p_core, nm_gate_t p_gate, nm_core_tag_t tag, nm_seq_t seq,
                          nm_len_t chunk_offset, nm_len_t chunk_len, int is_last_chunk,
                          nm_injector_pull_data_t p_pull_data, void*p_ref);
 
void nm_core_inject_complete(struct nm_core*p_core, struct nm_req_s*p_req, nm_len_t chunk_offset, nm_len_t chunk_len);
 
void nm_core_inject_finalize(struct nm_core*p_core, struct nm_req_s*p_req);
 
void nm_core_inject_complete_finalize(struct nm_core*p_core, struct nm_req_s*p_req, nm_len_t chunk_offset, nm_len_t chunk_len);
 
/* ** partitioned unpack */
 
void nm_core_unpack_partition_set(struct nm_req_s*p_unpack, int n_partitions);
 
void nm_core_unpack_partition_free(struct nm_req_s*p_unpack);
 
int nm_core_unpack_partition_test(struct nm_req_s*p_unpack, int partition);
 
/* ** synchronization primitives *************************** */
 
static inline void nm_cond_init(nm_cond_status_t*p_cond, nm_status_t bitmask);
 
static inline void nm_cond_destroy(nm_cond_status_t*p_cond);
 
static inline nm_status_t nm_cond_test(const nm_cond_status_t*p_cond, nm_status_t bitmask);
 
static inline nm_status_t nm_cond_test_locked(const nm_cond_status_t*p_cond, nm_status_t bitmask);
 
static inline void nm_cond_add(nm_cond_status_t*p_cond, nm_status_t bitmask);
 
static inline void nm_cond_wait(nm_cond_status_t*p_cond, nm_status_t bitmask, nm_core_t p_core);
 
static inline void nm_cond_signal(nm_cond_status_t*p_cond, nm_status_t bitmask);
 
static inline void nm_cond_wait_all(void**pp_conds, int n, uintptr_t offset, nm_status_t bitmask, nm_core_t p_core);
 
#if defined(PIOMAN)
 
static inline void nm_cond_init(nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  piom_cond_init(p_cond, bitmask);
}
static inline void nm_cond_destroy(nm_cond_status_t*p_cond)
{
  piom_cond_destroy(p_cond);
}
static inline nm_status_t nm_cond_test(const nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  return piom_cond_test(p_cond, bitmask);
}
static inline nm_status_t nm_cond_test_locked(const nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  return piom_cond_test_locked((nm_cond_status_t*)p_cond, bitmask);
}
static inline void nm_cond_add(nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  piom_cond_add(p_cond, bitmask);
}
static inline void nm_cond_mask(nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  piom_cond_mask(p_cond, bitmask);
}
static inline void nm_cond_wait(nm_cond_status_t*p_cond, nm_status_t bitmask, nm_core_t p_core __attribute__((unused)))
{
  piom_cond_wait(p_cond, bitmask);
}
static inline void nm_cond_signal(nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  piom_cond_signal(p_cond, bitmask);
}
static inline void nm_cond_wait_all(void**pp_conds, int n, uintptr_t offset, nm_status_t bitmask, nm_core_t p_core  __attribute__((unused)))
{
  piom_cond_wait_all(pp_conds, n, offset, bitmask);
}
#else /* PIOMAN */
static inline void nm_cond_init(nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  *p_cond = bitmask;
}
static inline void nm_cond_destroy(nm_cond_status_t*p_cond __attribute__((unused)))
{
}
static inline nm_status_t nm_cond_test(const nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  return ((*p_cond) & bitmask);
}
static inline nm_status_t nm_cond_test_locked(const nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  return nm_cond_test(p_cond, bitmask);
}
static inline void nm_cond_add(nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  *p_cond |= bitmask;
}
static inline void nm_cond_mask(nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  *p_cond &= ~bitmask;
}
static inline void nm_cond_signal(nm_cond_status_t*p_cond, nm_status_t bitmask)
{
  *p_cond |= bitmask;
}
static inline void nm_cond_wait(nm_cond_status_t*p_cond, nm_status_t bitmask, nm_core_t p_core)
{
  while(!nm_cond_test(p_cond, bitmask))
    {
      nm_schedule(p_core);
    }
}
static inline void nm_cond_wait_all(void**pp_conds, int n, uintptr_t offset, nm_status_t bitmask, nm_core_t p_core)
{
  int i;
  for(i = 0; i < n; i++)
    {
      if(pp_conds[i] != NULL)
        {
          nm_cond_status_t*p_cond = (nm_cond_status_t*)((uintptr_t)pp_conds[i] + offset);
          nm_cond_wait(p_cond, bitmask, p_core);
        }
    }
}
#endif /* PIOMAN */
 
/* ** convenient frontends to deal with status in requests */
 
static inline void nm_status_init(struct nm_req_s*p_req, nm_status_t bitmask)
{
  nm_cond_init(&p_req->status, bitmask);
}
static inline void nm_status_destroy(struct nm_req_s*p_req)
{
  nm_cond_destroy(&p_req->status);
}
static inline nm_status_t nm_status_test(const struct nm_req_s*p_req, nm_status_t bitmask)
{
  if(bitmask & NM_STATUS_FINALIZED) /* status FINALIZED needs strong consistency to avoid use after free */
    return nm_cond_test_locked(&p_req->status, bitmask);
  else
    return nm_cond_test(&p_req->status, bitmask);
}
static inline void nm_status_add(struct nm_req_s*p_req, nm_status_t bitmask)
{
  nm_cond_add(&p_req->status, bitmask);
}
static inline void nm_status_unset(struct nm_req_s*p_req, nm_status_t bitmask)
{
  nm_cond_mask(&p_req->status, bitmask);
}
static inline void nm_status_wait(struct nm_req_s*p_req, nm_status_t bitmask, nm_core_t p_core)
{
  nm_cond_wait(&p_req->status, bitmask, p_core);
  assert(nm_status_test(p_req, bitmask) != 0);
}
static inline void nm_status_signal(struct nm_req_s*p_req, nm_status_t bitmask)
{
  nm_cond_signal(&p_req->status, bitmask);
}
static inline void nm_status_wait_all(void**pp_reqs, int n, uintptr_t offset,
                                      nm_status_t bitmask, nm_core_t p_core)
{
  const struct nm_req_s*p_req = NULL;
  const uintptr_t status_offset = (uintptr_t)&p_req->status - (uintptr_t)p_req; /* offset of 'status' in nm_req_s */
  nm_cond_wait_all(pp_reqs, n, offset + status_offset, bitmask, p_core);
}
static inline void nm_status_assert(struct nm_req_s*p_req __attribute__((unused)), nm_status_t value __attribute__((unused)))
{
  assert(nm_status_test(p_req, NM_STATUS_MASK_FULL) == value);
}
 
static inline void nm_status_spinwait(struct nm_req_s*p_req, nm_status_t status)
{
  while(!nm_status_test(p_req, status))
    { /* bust wait*/ }
}
static inline int nm_status_test_allbits(struct nm_req_s*p_req, nm_status_t bitmask)
{
  return (nm_status_test(p_req, bitmask) == bitmask);
}
 
/* ** frontends for atomic ops ***************************** */
 
static inline void nm_mem_fence_always(void)
{
  __sync_synchronize();
}
 
static inline void nm_mem_fence(void)
{
#if defined(PIOMAN_MULTITHREAD)
  __sync_synchronize();
#else
  nm_core_t p_core = nm_core_get_singleton();
  if(nm_core_get_thread_level(p_core) >= NM_THREAD_SERIALIZED)
    {
      __sync_synchronize();
    }
#endif /* PIOMAN_MULTITHREAD */
}
 
static inline int nm_atomic_inc(int*v)
{
#if defined(PIOMAN_MULTITHREAD)
  return __sync_fetch_and_add(v, 1);
#else
  nm_core_t p_core = nm_core_get_singleton();
  if(nm_core_get_thread_level(p_core) >= NM_THREAD_SERIALIZED)
    {
      return __sync_fetch_and_add(v, 1);
    }
  else
    {
      return (*v)++;
    }
#endif /* PIOMAN_MULTITHREAD */
}
 
static inline int nm_atomic_alway_inc(int*v)
{
  return __sync_fetch_and_add(v, 1);
}
 
static inline int nm_atomic_dec(int*v)
{
#if defined(PIOMAN_MULTITHREAD)
  return __sync_sub_and_fetch(v, 1);
#else
  nm_core_t p_core = nm_core_get_singleton();
  if(nm_core_get_thread_level(p_core) >= NM_THREAD_SERIALIZED)
    {
      return __sync_sub_and_fetch(v, 1);
    }
  else
    {
      return --(*v);
    }
#endif /* PIOMAN_MULTITHREAD */
}
 
static inline int nm_atomic_always_dec(int*v)
{
  return __sync_sub_and_fetch(v, 1);
}
 
static inline void nm_atomic_add(int*v, int v2)
{
#if defined(PIOMAN_MULTITHREAD)
  __sync_fetch_and_add(v, v2);
#else
  nm_core_t p_core = nm_core_get_singleton();
  if(nm_core_get_thread_level(p_core) >= NM_THREAD_SERIALIZED)
    {
      __sync_fetch_and_add(v, v2);
    }
  else
    {
      (*v) += v2;
    }
#endif /* PIOMAN_MULTITHREAD */
}
 
static inline void nm_atomic_always_add(int*v, int v2)
{
  __sync_fetch_and_add(v, v2);
}
 
static inline int nm_atomic_compare_and_swap(int*v, int oldval, int newval)
{
#if defined(PIOMAN_MULTITHREAD)
  return __sync_bool_compare_and_swap(v, oldval, newval);
#else
  nm_core_t p_core = nm_core_get_singleton();
  if(nm_core_get_thread_level(p_core) >= NM_THREAD_SERIALIZED)
    {
      return __sync_bool_compare_and_swap(v, oldval, newval);
    }
  else
    {
      if(*v == oldval)
        {
          *v = newval;
          return 1;
        }
      else
        {
          return 0;
        }
    }
#endif /* PIOMAN_MULTITHREAD */
}
 
static inline int nm_atomic_always_compare_and_swap(int*v, int oldval, int newval)
{
  return __sync_bool_compare_and_swap(v, oldval, newval);
}
 
/* ** frontend for generic locking ************************* */
 
#ifdef PIOMAN
typedef piom_spinlock_t nm_spinlock_t;
#else /* PIOMAN */
struct nm_spinlock_s
{
#ifdef NMAD_DEBUG
  int lock;
  pthread_t last_tid;
#endif /* NMAD_DEBUG */
};
typedef struct nm_spinlock_s nm_spinlock_t;
#endif /* PIOMAN */
 
static inline void nm_spin_init(nm_spinlock_t*p_spin);
 
static inline void nm_spin_destroy(nm_spinlock_t*p_spin);
 
static inline void nm_spin_lock(nm_spinlock_t*p_spin);
 
static inline void nm_spin_unlock(nm_spinlock_t*p_spin);
 
static inline int nm_spin_trylock(nm_spinlock_t*p_spin);
 
static inline void nm_spin_assert_locked(nm_spinlock_t*p_spin);
 
static inline void nm_spin_assert_notlocked(nm_spinlock_t*p_spin);
 
 
static inline void nm_spin_check_nothread(nm_spinlock_t*p_spin __attribute__((unused)))
{
#if defined(NMAD_DEBUG) && !defined(PIOMAN)
  nm_core_t p_core = nm_core_get_singleton();
  assert(nm_core_get_thread_level(p_core) <= NM_THREAD_SERIALIZED);
  if(p_spin->last_tid == (pthread_t)0)
    {
      p_spin->last_tid = pthread_self();
      __sync_synchronize();
    }
  else
    {
      if(p_spin->last_tid != pthread_self())
        {
          NM_FATAL("detected calls from multiple threads in non-threaded mode. Please use pioman-enabled build for multi-threaded use or give thread level using nm_core_set_thread_level(NM_THREAD_SERIALIZED) for serialized thread level.");
        }
    }
#endif /* NMAD_DEBUG && !PIOMAN */
}
 
static inline void nm_spin_clear_nothread(nm_spinlock_t*p_spin __attribute__((unused)))
{
#if defined(NMAD_DEBUG) && !defined(PIOMAN)
  nm_core_t p_core = nm_core_get_singleton();
  assert(nm_core_get_thread_level(p_core) <= NM_THREAD_SERIALIZED);
  if(p_spin->last_tid == 0)
    {
      NM_FATAL("unlocking while no thread is holding the lock.");
    }
  else if(p_spin->last_tid != pthread_self())
    {
      NM_WARN("unlocking from another thread than where lock was acquired.\n");
    }
  if(nm_core_get_thread_level(p_core) == NM_THREAD_SERIALIZED)
    {
      p_spin->last_tid = (pthread_t)0;
    }
#endif /* NMAD_DEBUG && !PIOMAN */
}
 
static inline void nm_spin_init(nm_spinlock_t*p_spin __attribute__((unused)))
{
#ifdef PIOMAN
  piom_spin_init(p_spin);
#else /* PIOMAN */
#ifdef NMAD_DEBUG
  p_spin->lock = 0;
  p_spin->last_tid = 0;
#endif /* NMAD_DEBUG */
#endif /* PIOMAN */
}
 
static inline void nm_spin_destroy(nm_spinlock_t*p_spin __attribute__((unused)))
{
#ifdef PIOMAN
  piom_spin_destroy(p_spin);
#else /* PIOMAN */
#ifdef NMAD_DEBUG
  assert(p_spin->lock == 0);
#endif /* NMAD_DEBUG */
#endif /* PIOMAN */
}
 
static inline void nm_spin_lock(nm_spinlock_t*p_spin __attribute__((unused)))
{
#ifdef PIOMAN
  piom_spin_lock(p_spin);
#else /* PIOMAN */
#ifdef NMAD_DEBUG
  __sync_synchronize();
  if(p_spin->lock != 0)
    {
      NM_FATAL("spinlock is not free in nm_spin_lock(); detected concurrent access from thread = %p. Suspecting multi-threaded use by the application while library is initialized in non-threaded mode.\n",
               (void*)p_spin->last_tid);
    }
  p_spin->lock = 1;
#endif /* NMAD_DEBUG */
  nm_spin_check_nothread(p_spin);
  nm_core_t p_core = nm_core_get_singleton();
  if(nm_core_get_thread_level(p_core) >= NM_THREAD_SERIALIZED)
    {
      __sync_synchronize();
    }
#endif /* PIOMAN */
}
 
static inline void nm_spin_unlock(nm_spinlock_t*p_spin __attribute__((unused)))
{
#ifdef PIOMAN
  piom_spin_unlock(p_spin);
#else /* PIOMAN */
  nm_spin_clear_nothread(p_spin);
#ifdef NMAD_DEBUG
  __sync_synchronize();
  assert(p_spin->lock == 1);
  p_spin->lock = 0;
#endif /* NMAD_DEBUG */
  nm_core_t p_core = nm_core_get_singleton();
  if(nm_core_get_thread_level(p_core) >= NM_THREAD_SERIALIZED)
    {
      __sync_synchronize();
    }
#endif /* PIOMAN */
}
 
static inline int nm_spin_trylock(nm_spinlock_t*p_spin __attribute__((unused)))
{
#ifdef PIOMAN
  return piom_spin_trylock(p_spin);
#else /* PIOMAN */
  int rc = 1;
#ifdef NMAD_DEBUG
  __sync_synchronize();
  if(p_spin->lock)
    {
      assert(p_spin->lock == 1);
      rc = 0;
    }
  else
    {
      rc = 1;
      p_spin->lock = 1;
      nm_spin_check_nothread(p_spin);
    }
#endif /* NMAD_DEBUG */
  nm_core_t p_core = nm_core_get_singleton();
  if(nm_core_get_thread_level(p_core) >= NM_THREAD_SERIALIZED)
    {
      __sync_synchronize();
    }
  return rc;
#endif /* PIOMAN */
}
 
static inline void nm_spin_assert_locked(nm_spinlock_t*p_spin __attribute__((unused)))
{
#ifdef PIOMAN
  piom_spin_assert_locked(p_spin);
#else /* PIOMAN */
#ifdef NMAD_DEBUG
  assert(p_spin->lock == 1);
  assert(p_spin->last_tid == pthread_self());
#endif /* NMAD_DEBUG */
#endif /* PIOMAN */
}
 
static inline void nm_spin_assert_notlocked(nm_spinlock_t*p_spin __attribute__((unused)))
{
#ifdef PIOMAN
  piom_spin_assert_notlocked(p_spin);
#else /* PIOMAN */
#ifdef NMAD_DEBUG
  assert(p_spin->lock == 0);
#endif /* NMAD_DEBUG */
#endif /* PIOMAN */
}
 
#endif /* NM_CORE_INTERFACE_H */