/* $Source: bitbucket.org:berkeleylab/gasnet.git/extended-ref/gasnet_extended.h $ * Description: GASNet Extended API Header * Copyright 2002, Dan Bonachea * Terms of use are as specified in license.txt */ #ifndef _IN_GASNETEX_H #error This file is not meant to be included directly- clients should include gasnetex.h #endif #ifndef _GASNET_EXTENDED_H #define _GASNET_EXTENDED_H #include #include /* TODO: add debug code to enforce restrictions on SEQ and PARSYNC config */ /* (only one thread calls, HSL's only locked by that thread - how to check without pthread_getspecific()?) */ /* ------------------------------------------------------------------------------------ */ /* Initialization ============== */ /* Initialize the Extended API: must be called by the core API at the end of gasnet_attach() before calls to extended API (this function may make calls to the core functions) */ extern void gasnete_init(void); /* ------------------------------------------------------------------------------------ */ /* Non-blocking memory-to-memory transfers (explicit event) ========================================================== */ #ifndef gasnete_put_nb extern gex_Event_t gasnete_put_nb( gex_TM_t _tm, gex_Rank_t _rank, void *_dest, /*const*/ void *_src, // TODO-EX: un-comment const size_t _nbytes, gex_Event_t *_lc_opt, gex_Flags_t _flags GASNETI_THREAD_FARG) GASNETI_WARN_UNUSED_RESULT; #endif #ifndef gasnete_get_nb extern gex_Event_t gasnete_get_nb( gex_TM_t _tm, void *_dest, gex_Rank_t _rank, void *_src, size_t _nbytes, gex_Flags_t _flags GASNETI_THREAD_FARG) GASNETI_WARN_UNUSED_RESULT; #endif GASNETI_INLINE(_gex_RMA_GetNB) GASNETI_WARN_UNUSED_RESULT gex_Event_t _gex_RMA_GetNB( gex_TM_t _tm, void *_dest, gex_Rank_t _rank, void *_src, size_t _nbytes, gex_Flags_t _flags, gex_Flags_t _constantp_never_nbrhd GASNETI_THREAD_FARG) { GASNETI_CHECK_LEGACY_GET(_tm,_flags); GASNETI_CHECK_NEVER_FLAGS(_tm,_rank,_flags); GASNETI_CHECKZEROSZ_GET(NB,_tm,_dest,_rank,_src,_nbytes); gasneti_boundscheck(_tm, _rank, _src, _nbytes); gasneti_boundscheck_local(_tm, _dest, _nbytes); void *_mapped_src = GASNETE_NEVER_MAPPED(_flags, _constantp_never_nbrhd) ? NULL : gasnete_mapped_at(_tm, _rank, _src); if (_mapped_src) { GASNETI_TRACE_GET_LOCAL(NB,_tm,_dest,_rank,_src,_nbytes); GASNETI_CHECK_INJECT(); GASNETI_MEMCPY(_dest, _mapped_src, _nbytes); gasnete_loopbackget_memsync(); return GEX_EVENT_INVALID; } else { GASNETI_TRACE_GET(NB,_tm,_dest,_rank,_src,_nbytes); GASNETI_CHECK_INJECT(); return gasnete_get_nb(_tm, _dest, _rank, _src, _nbytes, _flags GASNETI_THREAD_PASS); } } #define gex_RMA_GetNB(tm,dest,rank,src,nbytes,flags) \ _gex_RMA_GetNB(tm,dest,rank,src,nbytes,flags, \ GASNETE_CONSTANTP_NEVER_NBRHD(flags) \ GASNETI_THREAD_GET) GASNETI_INLINE(_gex_RMA_PutNB) GASNETI_WARN_UNUSED_RESULT gex_Event_t _gex_RMA_PutNB( gex_TM_t _tm, gex_Rank_t _rank, void *_dest, /*const*/ void *_src, // TODO-EX: un-comment const size_t _nbytes, gex_Event_t *_lc_opt, gex_Flags_t _flags, gex_Flags_t _constantp_never_nbrhd GASNETI_THREAD_FARG) { GASNETI_CHECK_LEGACY_PUT(_tm,_flags); GASNETI_CHECK_NEVER_FLAGS(_tm,_rank,_flags); GASNETI_CHECK_PUT_LCOPT(_lc_opt, 0); GASNETI_CHECKZEROSZ_PUT(NB,_tm,_rank,_dest,_src,_nbytes); gasneti_boundscheck(_tm, _rank, _dest, _nbytes); gasneti_boundscheck_local(_tm, _src, _nbytes); void *_mapped_dest = GASNETE_NEVER_MAPPED(_flags, _constantp_never_nbrhd) ? NULL : gasnete_mapped_at(_tm, _rank, _dest); if (_mapped_dest) { GASNETI_TRACE_PUT_LOCAL(NB,_tm,_rank,_dest,_src,_nbytes); GASNETI_CHECK_INJECT(); GASNETI_MEMCPY(_mapped_dest, _src, _nbytes); gasnete_loopbackput_memsync(); gasneti_leaf_finish(_lc_opt); return GEX_EVENT_INVALID; } else { GASNETI_TRACE_PUT(NB,_tm,_rank,_dest,_src,_nbytes); GASNETI_CHECK_INJECT(); return gasnete_put_nb(_tm, _rank, _dest, _src, _nbytes, _lc_opt, _flags GASNETI_THREAD_PASS); } } #define gex_RMA_PutNB(tm,rank,dest,src,nbytes,lc_opt,flags) \ _gex_RMA_PutNB(tm,rank,dest,src,nbytes,lc_opt,flags, \ GASNETE_CONSTANTP_NEVER_NBRHD(flags) \ GASNETI_THREAD_GET) /* ------------------------------------------------------------------------------------ */ /* Synchronization for explicit-event non-blocking operations: =========================================================== */ #ifndef gasnete_test extern int gasnete_test(gex_Event_t _event GASNETI_THREAD_FARG); #endif #ifndef gasnete_test_some extern int gasnete_test_some(gex_Event_t *_pevent, size_t _numevents GASNETI_THREAD_FARG); #endif #ifndef gasnete_test_all extern int gasnete_test_all (gex_Event_t *_pevent, size_t _numevents GASNETI_THREAD_FARG); #endif GASNETI_INLINE(_gex_Event_Test) GASNETI_WARN_UNUSED_RESULT int _gex_Event_Test(gex_Event_t _event GASNETI_THREAD_FARG) { GASNETI_CHECK_INJECT(); int _result = GASNET_OK; if_pt (_event != GEX_EVENT_INVALID) _result = gasnete_test(_event GASNETI_THREAD_PASS); GASNETI_TRACE_TRYSYNC(TEST_SYNCNB,_result); return _result; } #define gex_Event_Test(pevent) \ _gex_Event_Test(pevent GASNETI_THREAD_GET) GASNETI_INLINE(_gex_Event_TestSome) int _gex_Event_TestSome(gex_Event_t *_pevent, size_t _numevents, gex_Flags_t _flags GASNETI_THREAD_FARG) { GASNETI_CHECK_INJECT(); int _result = gasnete_test_some(_pevent,_numevents GASNETI_THREAD_PASS); GASNETI_TRACE_TRYSYNC(TEST_SYNCNB_SOME,_result); return _result; } #define gex_Event_TestSome(pevent, numevents, flags) \ _gex_Event_TestSome(pevent, numevents, flags GASNETI_THREAD_GET) GASNETI_INLINE(_gex_Event_TestAll) int _gex_Event_TestAll(gex_Event_t *_pevent, size_t _numevents, gex_Flags_t _flags GASNETI_THREAD_FARG) { GASNETI_CHECK_INJECT(); int _result = gasnete_test_all(_pevent,_numevents GASNETI_THREAD_PASS); GASNETI_TRACE_TRYSYNC(TEST_SYNCNB_ALL,_result); return _result; } #define gex_Event_TestAll(pevent, numevents, flags) \ _gex_Event_TestAll(pevent, numevents, flags GASNETI_THREAD_GET) #ifndef gasnete_wait GASNETI_INLINE(gasnete_wait) void gasnete_wait(gex_Event_t _event GASNETI_THREAD_FARG) { if_pt (_event != GEX_EVENT_INVALID) { gasneti_AMPoll(); /* Ensure at least one poll - TODO: remove? */ gasneti_pollwhile(gasnete_test(_event GASNETI_THREAD_PASS) == GASNET_ERR_NOT_READY); } } #endif GASNETI_INLINE(_gex_Event_Wait) void _gex_Event_Wait(gex_Event_t _event GASNETI_THREAD_FARG) { GASNETI_CHECK_INJECT(); GASNETI_TRACE_WAITSYNC_BEGIN(); gasnete_wait(_event GASNETI_THREAD_PASS); GASNETI_TRACE_WAITSYNC_END(WAIT_SYNCNB); } #define gex_Event_Wait(event) \ _gex_Event_Wait(event GASNETI_THREAD_GET) #ifndef gasnete_wait_some // TODO-EX: a non-inline function could allow some optimizations GASNETI_INLINE(gasnete_wait_some) void gasnete_wait_some(gex_Event_t *_pevent, size_t _numevents GASNETI_THREAD_FARG) { gasneti_AMPoll(); /* Ensure at least one poll - TODO: remove? */ gasneti_pollwhile(gasnete_test_some(_pevent, _numevents GASNETI_THREAD_PASS) == GASNET_ERR_NOT_READY); } #endif GASNETI_INLINE(_gex_Event_WaitSome) void _gex_Event_WaitSome(gex_Event_t *_pevent, size_t _numevents, gex_Flags_t _flags GASNETI_THREAD_FARG) { GASNETI_CHECK_INJECT(); GASNETI_TRACE_WAITSYNC_BEGIN(); gasnete_wait_some(_pevent, _numevents GASNETI_THREAD_PASS); GASNETI_TRACE_WAITSYNC_END(WAIT_SYNCNB_SOME); } #define gex_Event_WaitSome(pevent, numevents, flags) \ _gex_Event_WaitSome(pevent, numevents, flags GASNETI_THREAD_GET) #ifndef gasnete_wait_all // TODO-EX: a non-inline function could allow some optimizations GASNETI_INLINE(gasnete_wait_all) void gasnete_wait_all(gex_Event_t *_pevent, size_t _numevents GASNETI_THREAD_FARG) { GASNETI_CHECK_INJECT(); gasneti_AMPoll(); /* Ensure at least one poll - TODO: remove? */ gasneti_pollwhile(gasnete_test_all(_pevent, _numevents GASNETI_THREAD_PASS) == GASNET_ERR_NOT_READY); } #endif GASNETI_INLINE(_gex_Event_WaitAll) void _gex_Event_WaitAll(gex_Event_t *_pevent, size_t _numevents, gex_Flags_t _flags GASNETI_THREAD_FARG) { GASNETI_CHECK_INJECT(); GASNETI_TRACE_WAITSYNC_BEGIN(); gasnete_wait_all(_pevent, _numevents GASNETI_THREAD_PASS); GASNETI_TRACE_WAITSYNC_END(WAIT_SYNCNB_ALL); } #define gex_Event_WaitAll(pevent, numevents, flags) \ _gex_Event_WaitAll(pevent, numevents, flags GASNETI_THREAD_GET) /* ------------------------------------------------------------------------------------ */ /* Non-blocking memory-to-memory transfers (implicit event) ========================================================== */ #ifndef gasnete_get_nbi extern int gasnete_get_nbi (gex_TM_t _tm, void *_dest, gex_Rank_t _rank, void *_src, size_t _nbytes, gex_Flags_t _flags GASNETI_THREAD_FARG); #endif #ifndef gasnete_put_nbi extern int gasnete_put_nbi (gex_TM_t _tm, gex_Rank_t _rank, void *_dest, /*const*/ void *_src, // TODO-EX: un-comment const size_t _nbytes, gex_Event_t *_lc_opt, gex_Flags_t _flags GASNETI_THREAD_FARG); #endif GASNETI_INLINE(_gex_RMA_GetNBI) int _gex_RMA_GetNBI (gex_TM_t _tm, void *_dest, gex_Rank_t _rank, void *_src, size_t _nbytes, gex_Flags_t _flags, gex_Flags_t _constantp_never_nbrhd GASNETI_THREAD_FARG) { GASNETI_CHECK_LEGACY_GET(_tm,_flags); GASNETI_CHECK_NEVER_FLAGS(_tm,_rank,_flags); GASNETI_CHECKZEROSZ_GET(NBI,_tm,_dest,_rank,_src,_nbytes); gasneti_boundscheck(_tm, _rank, _src, _nbytes); gasneti_boundscheck_local(_tm, _dest, _nbytes); void *_mapped_src = GASNETE_NEVER_MAPPED(_flags, _constantp_never_nbrhd) ? NULL : gasnete_mapped_at(_tm, _rank, _src); if (_mapped_src) { GASNETI_TRACE_GET_LOCAL(NBI,_tm,_dest,_rank,_src,_nbytes); GASNETI_CHECK_INJECT(); GASNETI_MEMCPY(_dest, _mapped_src, _nbytes); gasnete_loopbackget_memsync(); return 0; } else { GASNETI_TRACE_GET(NBI,_tm,_dest,_rank,_src,_nbytes); GASNETI_CHECK_INJECT(); return gasnete_get_nbi(_tm, _dest, _rank, _src, _nbytes, _flags GASNETI_THREAD_PASS); } } #define gex_RMA_GetNBI(tm,dest,rank,src,nbytes,flags) \ _gex_RMA_GetNBI(tm,dest,rank,src,nbytes,flags, \ GASNETE_CONSTANTP_NEVER_NBRHD(flags) \ GASNETI_THREAD_GET) GASNETI_INLINE(_gex_RMA_PutNBI) int _gex_RMA_PutNBI (gex_TM_t _tm, gex_Rank_t _rank, void *_dest, /*const*/ void *_src, // TODO-EX: un-comment const size_t _nbytes, gex_Event_t *_lc_opt, gex_Flags_t _flags, gex_Flags_t _constantp_never_nbrhd GASNETI_THREAD_FARG) { GASNETI_CHECK_LEGACY_PUT(_tm,_flags); GASNETI_CHECK_NEVER_FLAGS(_tm,_rank,_flags); GASNETI_CHECK_PUT_LCOPT(_lc_opt, 1); GASNETI_CHECKZEROSZ_PUT(NBI,_tm,_rank,_dest,_src,_nbytes); gasneti_boundscheck(_tm, _rank, _dest, _nbytes); gasneti_boundscheck_local(_tm, _src, _nbytes); void *_mapped_dest = GASNETE_NEVER_MAPPED(_flags, _constantp_never_nbrhd) ? NULL : gasnete_mapped_at(_tm, _rank, _dest); if (_mapped_dest) { GASNETI_TRACE_PUT_LOCAL(NBI,_tm,_rank,_dest,_src,_nbytes); GASNETI_CHECK_INJECT(); GASNETI_MEMCPY(_mapped_dest, _src, _nbytes); gasnete_loopbackput_memsync(); return 0; } else { GASNETI_TRACE_PUT(NBI,_tm,_rank,_dest,_src,_nbytes); GASNETI_CHECK_INJECT(); return gasnete_put_nbi(_tm, _rank, _dest, _src, _nbytes, _lc_opt, _flags GASNETI_THREAD_PASS); } } #define gex_RMA_PutNBI(tm,rank,dest,src,nbytes,lc_opt,flags) \ _gex_RMA_PutNBI(tm,rank,dest,src,nbytes,lc_opt,flags, \ GASNETE_CONSTANTP_NEVER_NBRHD(flags) \ GASNETI_THREAD_GET) /* ------------------------------------------------------------------------------------ */ /* Synchronization for implicit-event non-blocking operations: =========================================================== */ // The internal event categories #define GASNETI_EC_PUT (1U << 0) #define GASNETI_EC_ALC (1U << 1) #define GASNETI_EC_GET (1U << 2) #define GASNETI_EC_RMW (1U << 3) typedef unsigned int gex_EC_t; #ifndef GEX_EC_PUT #define GEX_EC_PUT ((gex_EC_t)(GASNETI_EC_PUT|GASNETI_EC_ALC)) #endif #ifndef GEX_EC_GET #define GEX_EC_GET ((gex_EC_t)GASNETI_EC_GET) #endif #ifndef GEX_EC_AM #define GEX_EC_AM ((gex_EC_t)GASNETI_EC_ALC) #endif #ifndef GEX_EC_LC #define GEX_EC_LC ((gex_EC_t)GASNETI_EC_ALC) #endif #ifndef GEX_EC_RMW #define GEX_EC_RMW ((gex_EC_t)GASNETI_EC_RMW) #endif #ifndef GEX_EC_ALL #define GEX_EC_ALL (~(gex_EC_t)0) #endif #ifndef gasnete_test_syncnbi_mask extern int gasnete_test_syncnbi_mask(gex_EC_t _mask, gex_Flags_t _flags GASNETI_THREAD_FARG); #endif GASNETI_INLINE(_gex_NBI_Test) GASNETI_WARN_UNUSED_RESULT int _gex_NBI_Test(gex_EC_t _mask, gex_Flags_t _flags GASNETI_THREAD_FARG) { GASNETI_CHECK_INJECT(); int _retval = gasnete_test_syncnbi_mask(_mask, _flags GASNETI_THREAD_PASS); GASNETI_TRACE_TRYSYNC(TEST_SYNCNBI,_retval); return _retval; } #define gex_NBI_Test(mask, flags) \ _gex_NBI_Test(mask, flags GASNETI_THREAD_GET) #ifndef gasnete_wait_syncnbi_mask #define gasnete_wait_syncnbi_mask(mask, flagsTI) \ gasneti_pollwhile(gasnete_test_syncnbi_mask(mask, flagsTI) == GASNET_ERR_NOT_READY) #endif #define gex_NBI_Wait(mask, flags) do { \ GASNETI_CHECK_INJECT(); \ GASNETI_TRACE_WAITSYNC_BEGIN(); \ gasneti_AMPoll(); /* ensure at least one poll */ \ gasnete_wait_syncnbi_mask(mask, flags GASNETI_THREAD_GET); \ GASNETI_TRACE_WAITSYNC_END(WAIT_SYNCNBI); \ } while (0) /* ------------------------------------------------------------------------------------ */ /* Implicit access region synchronization ====================================== */ #ifndef gasnete_begin_nbi_accessregion extern void gasnete_begin_nbi_accessregion(gex_Flags_t _flags, int _allowrecursion GASNETI_THREAD_FARG); #endif #ifndef gasnete_end_nbi_accessregion extern gex_Event_t gasnete_end_nbi_accessregion(gex_Flags_t _flags GASNETI_THREAD_FARG) GASNETI_WARN_UNUSED_RESULT; #endif #define gex_NBI_BeginAccessRegion(flags) gasnete_begin_nbi_accessregion(flags,0 GASNETI_THREAD_GET) #define gex_NBI_EndAccessRegion(flags) gasnete_end_nbi_accessregion(flags GASNETI_THREAD_GET) /* ------------------------------------------------------------------------------------ */ /* Query to get leaf event from its root ===================================== */ #ifndef gasnete_Event_QueryLeaf extern gex_Event_t gasnete_Event_QueryLeaf(gex_Event_t _root, gex_EC_t _event_id); #endif #define gex_Event_QueryLeaf(root, event_id) gasnete_Event_QueryLeaf(root, event_id) /* ------------------------------------------------------------------------------------ */ /* Blocking memory-to-memory transfers =================================== */ #if GASNETI_DIRECT_BLOCKING_GET extern int gasnete_get (gex_TM_t _tm, void *_dest, gex_Rank_t _rank, void *_src, size_t _nbytes, gex_Flags_t _flags GASNETI_THREAD_FARG); #elif !defined(gasnete_get) GASNETI_INLINE(gasnete_get) int gasnete_get (gex_TM_t _tm, void *_dest, gex_Rank_t _rank, void *_src, size_t _nbytes, gex_Flags_t _flags GASNETI_THREAD_FARG) { gex_Event_t h = gasnete_get_nb(_tm, _dest, _rank, _src, _nbytes, _flags GASNETI_THREAD_PASS); if (h == GEX_EVENT_NO_OP) return 1; else gasnete_wait(h GASNETI_THREAD_PASS); return 0; } #endif #if GASNETI_DIRECT_BLOCKING_PUT extern int gasnete_put (gex_TM_t _tm, gex_Rank_t _rank, void* _dest, /*const*/ void *_src, // TODO-EX: uncomment const size_t _nbytes, gex_Flags_t _flags GASNETI_THREAD_FARG); #elif !defined(gasnete_put) GASNETI_INLINE(gasnete_put) int gasnete_put (gex_TM_t _tm, gex_Rank_t _rank, void* _dest, /*const*/ void *_src, // TODO-EX: uncomment const size_t _nbytes, gex_Flags_t _flags GASNETI_THREAD_FARG) { gex_Event_t h = gasnete_put_nb(_tm, _rank, _dest, _src, _nbytes, GEX_EVENT_DEFER, _flags GASNETI_THREAD_PASS); if (h == GEX_EVENT_NO_OP) return 1; else gasnete_wait(h GASNETI_THREAD_PASS); return 0; } #endif GASNETI_INLINE(_gex_RMA_GetBlocking) int _gex_RMA_GetBlocking (gex_TM_t _tm, void *_dest, gex_Rank_t _rank, void *_src, size_t _nbytes, gex_Flags_t _flags, gex_Flags_t _constantp_never_nbrhd GASNETI_THREAD_FARG) { GASNETI_CHECK_LEGACY_GET(_tm,_flags); GASNETI_CHECK_NEVER_FLAGS(_tm,_rank,_flags); GASNETI_CHECKZEROSZ_NAMED(GASNETI_TRACE_GET_NAMED(GET_LOCAL,LOCAL,_tm,_dest,_rank,_src,_nbytes),_nbytes); gasneti_boundscheck(_tm, _rank, _src, _nbytes); gasneti_boundscheck_local(_tm, _dest, _nbytes); void *_mapped_src = GASNETE_NEVER_MAPPED(_flags, _constantp_never_nbrhd) ? NULL : gasnete_mapped_at(_tm, _rank, _src); if (_mapped_src) { GASNETI_TRACE_GET_NAMED(GET_LOCAL,LOCAL,_tm,_dest,_rank,_src,_nbytes); GASNETI_CHECK_INJECT(); GASNETI_MEMCPY(_dest, _mapped_src, _nbytes); gasnete_loopbackget_memsync(); return 0; } else { GASNETI_TRACE_GET_NAMED(GET,NONLOCAL,_tm,_dest,_rank,_src,_nbytes); GASNETI_CHECK_INJECT(); return gasnete_get(_tm, _dest, _rank, _src, _nbytes, _flags GASNETI_THREAD_PASS); } } #define gex_RMA_GetBlocking(tm,dest,rank,src,nbytes,flags) \ _gex_RMA_GetBlocking(tm,dest,rank,src,nbytes,flags, \ GASNETE_CONSTANTP_NEVER_NBRHD(flags) \ GASNETI_THREAD_GET) GASNETI_INLINE(_gex_RMA_PutBlocking) int _gex_RMA_PutBlocking (gex_TM_t _tm, gex_Rank_t _rank, void *_dest, /*const*/ void *_src, // TODO-EX: un-comment const size_t _nbytes, gex_Flags_t _flags, gex_Flags_t _constantp_never_nbrhd GASNETI_THREAD_FARG) { GASNETI_CHECK_LEGACY_PUT(_tm,_flags); GASNETI_CHECK_NEVER_FLAGS(_tm,_rank,_flags); GASNETI_CHECKZEROSZ_NAMED(GASNETI_TRACE_PUT_NAMED(PUT_LOCAL,LOCAL,_tm,_rank,_dest,_src,_nbytes),_nbytes); gasneti_boundscheck(_tm, _rank, _dest, _nbytes); gasneti_boundscheck_local(_tm, _src, _nbytes); void *_mapped_dest = GASNETE_NEVER_MAPPED(_flags, _constantp_never_nbrhd) ? NULL : gasnete_mapped_at(_tm, _rank, _dest); if (_mapped_dest) { GASNETI_TRACE_PUT_NAMED(PUT_LOCAL,LOCAL,_tm,_rank,_dest,_src,_nbytes); GASNETI_CHECK_INJECT(); GASNETI_MEMCPY(_mapped_dest, _src, _nbytes); gasnete_loopbackput_memsync(); return 0; } else { GASNETI_TRACE_PUT_NAMED(PUT,NONLOCAL,_tm,_rank,_dest,_src,_nbytes); GASNETI_CHECK_INJECT(); return gasnete_put(_tm, _rank, _dest, _src, _nbytes, _flags GASNETI_THREAD_PASS); } } #define gex_RMA_PutBlocking(tm,rank,dest,src,nbytes,flags) \ _gex_RMA_PutBlocking(tm,rank,dest,src,nbytes,flags, \ GASNETE_CONSTANTP_NEVER_NBRHD(flags) \ GASNETI_THREAD_GET) /* ------------------------------------------------------------------------------------ */ /* Value Put ========= */ #if GASNETI_DIRECT_PUT_VAL extern int gasnete_put_val( gex_TM_t _tm, gex_Rank_t _rank, void *_dest, gex_RMA_Value_t _value, size_t _nbytes, gex_Flags_t _flags GASNETI_THREAD_FARG); #elif !defined(gasnete_put_val) GASNETI_INLINE(gasnete_put_val) int gasnete_put_val( gex_TM_t _tm, gex_Rank_t _rank, void *_dest, gex_RMA_Value_t _value, size_t _nbytes, gex_Flags_t _flags GASNETI_THREAD_FARG) { gasneti_assume(_nbytes > 0); gasneti_assume(_nbytes <= sizeof(gex_RMA_Value_t)); gex_RMA_Value_t _src = _value; return gasnete_put(_tm, _rank, _dest, GASNETE_STARTOFBITS(&_src,_nbytes), _nbytes, _flags GASNETI_THREAD_PASS); } #endif GASNETI_INLINE(_gex_RMA_PutVal) int _gex_RMA_PutVal( gex_TM_t _tm, gex_Rank_t _rank, void *_dest, gex_RMA_Value_t _value, size_t _nbytes, gex_Flags_t _flags, gex_Flags_t _constantp_never_nbrhd GASNETI_THREAD_FARG) { GASNETI_CHECK_LEGACY_PUT(_tm,_flags); GASNETI_CHECK_NEVER_FLAGS(_tm,_rank,_flags); gasneti_assume(_nbytes > 0); gasneti_assert_uint(_nbytes ,<=, sizeof(gex_RMA_Value_t)); gasneti_assume(_nbytes <= sizeof(gex_RMA_Value_t)); gasneti_boundscheck(_tm, _rank, _dest, _nbytes); void *_mapped_dest = GASNETE_NEVER_MAPPED(_flags, _constantp_never_nbrhd) ? NULL : gasnete_mapped_at(_tm, _rank, _dest); if (_mapped_dest) { GASNETI_TRACE_PUT_LOCAL(VAL,_tm,_rank,_dest,&_value,_nbytes); GASNETI_CHECK_INJECT(); GASNETE_VALUE_ASSIGN(_mapped_dest, _value, _nbytes); gasnete_loopbackput_memsync(); return 0; } else { GASNETI_TRACE_PUT(VAL,_tm,_rank,_dest,GASNETE_STARTOFBITS(&_value,_nbytes),_nbytes); GASNETI_CHECK_INJECT(); return gasnete_put_val(_tm, _rank, _dest, _value, _nbytes, _flags GASNETI_THREAD_PASS); } } #define gex_RMA_PutBlockingVal(tm,rank,dest,value,nbytes,flags) \ _gex_RMA_PutVal(tm,rank,dest,value,nbytes,flags, \ GASNETE_CONSTANTP_NEVER_NBRHD(flags) \ GASNETI_THREAD_GET) #if GASNETI_DIRECT_PUT_NB_VAL && !defined(gasnete_put_nb_val) extern gex_Event_t gasnete_put_nb_val( gex_TM_t _tm, gex_Rank_t _rank, void *_dest, gex_RMA_Value_t _value, size_t _nbytes, gex_Flags_t _flags GASNETI_THREAD_FARG) GASNETI_WARN_UNUSED_RESULT; #endif GASNETI_INLINE(_gex_RMA_PutNBVal) GASNETI_WARN_UNUSED_RESULT gex_Event_t _gex_RMA_PutNBVal ( gex_TM_t _tm, gex_Rank_t _rank, void *_dest, gex_RMA_Value_t _value, size_t _nbytes, gex_Flags_t _flags, gex_Flags_t _constantp_never_nbrhd GASNETI_THREAD_FARG) { GASNETI_CHECK_LEGACY_PUT(_tm,_flags); GASNETI_CHECK_NEVER_FLAGS(_tm,_rank,_flags); gasneti_assume(_nbytes > 0); gasneti_assert_uint(_nbytes ,<=, sizeof(gex_RMA_Value_t)); gasneti_assume(_nbytes <= sizeof(gex_RMA_Value_t)); gasneti_boundscheck(_tm, _rank, _dest, _nbytes); void *_mapped_dest = GASNETE_NEVER_MAPPED(_flags, _constantp_never_nbrhd) ? NULL : gasnete_mapped_at(_tm, _rank, _dest); if (_mapped_dest) { GASNETI_TRACE_PUT_LOCAL(NB_VAL,_tm,_rank,_dest,&_value,_nbytes); GASNETI_CHECK_INJECT(); GASNETE_VALUE_ASSIGN(_mapped_dest, _value, _nbytes); gasnete_loopbackput_memsync(); return GEX_EVENT_INVALID; } else { GASNETI_TRACE_PUT(NB_VAL,_tm,_rank,_dest,GASNETE_STARTOFBITS(&_value,_nbytes),_nbytes); GASNETI_CHECK_INJECT(); #if GASNETI_DIRECT_PUT_NB_VAL || defined(gasnete_put_nb_val) return gasnete_put_nb_val(_tm, _rank, _dest, _value, _nbytes, _flags GASNETI_THREAD_PASS); #else { gex_RMA_Value_t _src = _value; return gasnete_put_nb(_tm, _rank, _dest, GASNETE_STARTOFBITS(&_src,_nbytes), _nbytes, GEX_EVENT_NOW, _flags GASNETI_THREAD_PASS); } #endif } } #define gex_RMA_PutNBVal(tm,rank,dest,value,nbytes,flags) \ _gex_RMA_PutNBVal(tm,rank,dest,value,nbytes,flags, \ GASNETE_CONSTANTP_NEVER_NBRHD(flags) \ GASNETI_THREAD_GET) #if GASNETI_DIRECT_PUT_NBI_VAL extern int gasnete_put_nbi_val( gex_TM_t _tm, gex_Rank_t _rank, void *_dest, gex_RMA_Value_t _value, size_t _nbytes, gex_Flags_t _flags GASNETI_THREAD_FARG); #elif !defined(gasnete_put_nbi_val) GASNETI_INLINE(gasnete_put_nbi_val) int gasnete_put_nbi_val( gex_TM_t _tm, gex_Rank_t _rank, void *_dest, gex_RMA_Value_t _value, size_t _nbytes, gex_Flags_t _flags GASNETI_THREAD_FARG) { gasneti_assume(_nbytes > 0); gasneti_assume(_nbytes <= sizeof(gex_RMA_Value_t)); gex_RMA_Value_t _src = _value; return gasnete_put_nbi(_tm, _rank, _dest, GASNETE_STARTOFBITS(&_src,_nbytes), _nbytes, GEX_EVENT_NOW, _flags GASNETI_THREAD_PASS); } #endif GASNETI_INLINE(_gex_RMA_PutNBIVal) int _gex_RMA_PutNBIVal( gex_TM_t _tm, gex_Rank_t _rank, void *_dest, gex_RMA_Value_t _value, size_t _nbytes, gex_Flags_t _flags, gex_Flags_t _constantp_never_nbrhd GASNETI_THREAD_FARG) { GASNETI_CHECK_LEGACY_PUT(_tm,_flags); GASNETI_CHECK_NEVER_FLAGS(_tm,_rank,_flags); gasneti_assume(_nbytes > 0); gasneti_assert_uint(_nbytes ,<=, sizeof(gex_RMA_Value_t)); gasneti_assume(_nbytes <= sizeof(gex_RMA_Value_t)); gasneti_boundscheck(_tm, _rank, _dest, _nbytes); void *_mapped_dest = GASNETE_NEVER_MAPPED(_flags, _constantp_never_nbrhd) ? NULL : gasnete_mapped_at(_tm, _rank, _dest); if (_mapped_dest) { GASNETI_TRACE_PUT_LOCAL(NBI_VAL,_tm,_rank,_dest,&_value,_nbytes); GASNETI_CHECK_INJECT(); GASNETE_VALUE_ASSIGN(_mapped_dest, _value, _nbytes); gasnete_loopbackput_memsync(); return 0; } else { GASNETI_TRACE_PUT(NBI_VAL,_tm,_rank,_dest,GASNETE_STARTOFBITS(&_value,_nbytes),_nbytes); GASNETI_CHECK_INJECT(); return gasnete_put_nbi_val(_tm, _rank, _dest, _value, _nbytes, _flags GASNETI_THREAD_PASS); } } #define gex_RMA_PutNBIVal(tm,rank,dest,value,nbytes,flags) \ _gex_RMA_PutNBIVal(tm,rank,dest,value,nbytes,flags, \ GASNETE_CONSTANTP_NEVER_NBRHD(flags) \ GASNETI_THREAD_GET) /* ------------------------------------------------------------------------------------ */ /* Blocking Value Get ================== */ #if PLATFORM_COMPILER_SUN_C #pragma error_messages(off, E_END_OF_LOOP_CODE_NOT_REACHED) #endif #if !defined(gasnete_get_val) && GASNETI_DIRECT_GET_VAL extern gex_RMA_Value_t gasnete_get_val ( gex_TM_t _tm, gex_Rank_t _rank, void *_src, size_t _nbytes, gex_Flags_t _flags GASNETI_THREAD_FARG); #endif GASNETI_INLINE(_gex_RMA_GetBlockingVal) GASNETI_WARN_UNUSED_RESULT gex_RMA_Value_t _gex_RMA_GetBlockingVal ( gex_TM_t _tm, gex_Rank_t _rank, void *_src, size_t _nbytes, gex_Flags_t _flags, gex_Flags_t _constantp_never_nbrhd GASNETI_THREAD_FARG) { GASNETI_CHECK_LEGACY_GET(_tm,_flags); GASNETI_CHECK_NEVER_FLAGS(_tm,_rank,_flags); gasneti_boundscheck(_tm, _rank, _src, _nbytes); void *_mapped_src = GASNETE_NEVER_MAPPED(_flags, _constantp_never_nbrhd) ? NULL : gasnete_mapped_at(_tm, _rank, _src); if (_mapped_src) { GASNETI_TRACE_GET_LOCAL(VAL,_tm,NULL,_rank,_src,_nbytes); GASNETI_CHECK_INJECT(); GASNETE_VALUE_RETURN(_mapped_src, _nbytes); } else { GASNETI_TRACE_GET(VAL,_tm,NULL,_rank,_src,_nbytes); GASNETI_CHECK_INJECT(); #if GASNETI_DIRECT_GET_VAL || defined(gasnete_get_val) return gasnete_get_val(_tm, _rank, _src, _nbytes, _flags GASNETI_THREAD_PASS); #else { gex_RMA_Value_t _val = 0; gasnete_get(_tm, GASNETE_STARTOFBITS(&_val,_nbytes), _rank, _src, _nbytes, _flags GASNETI_THREAD_PASS); return _val; } #endif } } #define gex_RMA_GetBlockingVal(tm,rank,src,nbytes,flags) \ _gex_RMA_GetBlockingVal(tm,rank,src,nbytes,flags, \ GASNETE_CONSTANTP_NEVER_NBRHD(flags) \ GASNETI_THREAD_GET) #if PLATFORM_COMPILER_SUN_C #pragma error_messages(default, E_END_OF_LOOP_CODE_NOT_REACHED) #endif /* ------------------------------------------------------------------------------------ */ /* Barriers: ========= */ #if GASNETI_STATS_OR_TRACE extern gasneti_tick_t gasnete_barrier_notifytime; #endif extern void gasnet_barrier_notify(int _id, int _flags); extern int gasnet_barrier_wait(int _id, int _flags); extern int gasnet_barrier_try(int _id, int _flags); extern int gasnet_barrier(int _id, int _flags); extern int gasnet_barrier_result(int *_id); #ifndef gasnete_tm_barrier extern void gasnete_tm_barrier(gex_TM_t e_tm, gex_Flags_t flags GASNETI_THREAD_FARG); #endif GASNETI_INLINE(_gex_Coll_BarrierBlocking) void _gex_Coll_BarrierBlocking(gex_TM_t _tm, gex_Flags_t _flags GASNETI_THREAD_FARG) { GASNETI_TRACE_TM_BARRIER(COLL_BARRIER,_tm,_flags); GASNETI_CHECK_INJECT(); gasnete_tm_barrier(_tm, _flags GASNETI_THREAD_PASS); } #define gex_Coll_BarrierBlocking(tm,flags) \ _gex_Coll_BarrierBlocking(tm,flags GASNETI_THREAD_GET) #ifndef gasnete_tm_barrier_nb extern gex_Event_t gasnete_tm_barrier_nb(gex_TM_t e_tm, gex_Flags_t flags GASNETI_THREAD_FARG); #endif GASNETI_INLINE(_gex_Coll_BarrierNB) GASNETI_WARN_UNUSED_RESULT gex_Event_t _gex_Coll_BarrierNB(gex_TM_t _tm, gex_Flags_t _flags GASNETI_THREAD_FARG) { GASNETI_TRACE_TM_BARRIER(COLL_BARRIER_NB,_tm,_flags); GASNETI_CHECK_INJECT(); return gasnete_tm_barrier_nb(_tm, _flags GASNETI_THREAD_PASS); } #define gex_Coll_BarrierNB(tm,flags) \ _gex_Coll_BarrierNB(tm,flags GASNETI_THREAD_GET) /* ------------------------------------------------------------------------------------ */ /* Teams: ====== */ extern int gasneti_TM_Destroy( gex_TM_t _tm, gex_Memvec_t *_scratch_p, gex_Flags_t _flags GASNETI_THREAD_FARG); #define gex_TM_Destroy(tm,sp,fl) gasneti_TM_Destroy(tm,sp,fl GASNETI_THREAD_GET) extern size_t gasneti_TM_Split(gex_TM_t *_new_tm_p, gex_TM_t _parent_tm, int _color, int _key, void *_addr, size_t _len, gex_Flags_t _flags GASNETI_THREAD_FARG); #define gex_TM_Split(n,p,c,k,a,l,f) gasneti_TM_Split(n,p,c,k,a,l,f GASNETI_THREAD_GET) extern size_t gasneti_TM_Create( gex_TM_t *_new_tms, size_t _num_new_tms, gex_TM_t _parent_tm, gex_EP_Location_t *_args, size_t _numargs, gex_Addr_t *_scratch_addrs, size_t _scratch_size, gex_Flags_t _flags GASNETI_THREAD_FARG); #define gex_TM_Create(nt,nnt,p,a,n,sa,ss,f) gasneti_TM_Create(nt,nnt,p,a,n,sa,ss,f GASNETI_THREAD_GET) // extern gex_Rank_t gex_TM_TranslateRankToJobrank(gex_TM_t tm, gex_Rank_t rank); #define gex_TM_TranslateRankToJobrank(tm,rank) \ (GASNETI_CHECK_INJECT(), gasneti_e_tm_rank_to_jobrank(tm,rank)) // extern gex_Rank_t gex_TM_TranslateJobrankToRank(gex_TM_t tm, gex_Rank_t jobrank); #define gex_TM_TranslateJobrankToRank(tm,jobrank) \ (GASNETI_CHECK_INJECT(), gasneti_e_tm_jobrank_to_rank(tm,jobrank)) // extern gex_EP_Location_t gex_TM_TranslateRankToEP(gex_TM_t tm, gex_Rank_t rank, gex_Flags_t flags); #define gex_TM_TranslateRankToEP(tm,rank,flags) \ (GASNETI_CHECK_INJECT(), gasneti_e_tm_rank_to_location(tm,rank,flags)) /* ------------------------------------------------------------------------------------ */ // TODO-EX: remove these checks for conduits using legacy internal APIs #if GASNETI_DIRECT_GET #error "out-of-date #define of GASNETI_DIRECT_GET" #endif #if GASNETI_DIRECT_GET_BULK #error "out-of-date #define of GASNETI_DIRECT_GET_BULK" #elif defined(gasnete_get_bulk) #error "out-of-date #define of gasnete_get_bulk" #endif #if GASNETI_DIRECT_PUT #error "out-of-date #define of GASNETI_DIRECT_PUT" #endif #if GASNETI_DIRECT_PUT_BULK #error "out-of-date #define of GASNETI_DIRECT_PUT_BULK" #elif defined(gasnete_put_bulk) #error "out-of-date #define of gasnete_put_bulk" #endif #if GASNETI_DIRECT_GET_NBI #error "out-of-date #define of GASNETI_DIRECT_GET_NBI" #endif #if GASNETI_DIRECT_GET_NB #error "out-of-date #define of GASNETI_DIRECT_GET_NB" #endif #if GASNETI_DIRECT_MEMSET #error "out-of-date #define of GASNETI_DIRECT_MEMSET" #endif #endif