/* cilk_fiber.cpp -*-C++-*- * ************************************************************************* * * @copyright * Copyright (C) 2012-2013, Intel Corporation * All rights reserved. * * @copyright * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name of Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * @copyright * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. **************************************************************************/ /* Implementations of non-platform-specific aspects of cilk_fiber, especially * the cilk_fiber_pool interface. */ #include "cilk_fiber.h" #ifdef _WIN32 # include "cilk_fiber-win.h" #else # include "cilk_fiber-unix.h" #endif #include "cilk_malloc.h" #include "bug.h" #include #include #include #include #include #include "sysdep.h" extern "C" { inline int cilk_fiber_pool_sanity_check(cilk_fiber_pool *pool, const char* desc) { int errors = 0; #if FIBER_DEBUG >= 1 if ((NULL != pool) && pool->total > 0) { // Root pool should not allocate more fibers than alloc_max errors += ((pool->parent == NULL) && (pool->total > pool->alloc_max)); errors += (pool->total > pool->high_water); if (errors) { fprintf(stderr, "ERROR at %s: pool=%p has max_size=%u, total=%d, high_water=%d\n", desc, pool, pool->max_size, pool->total, pool->high_water); } } #endif return (errors == 0); } inline void increment_pool_total(cilk_fiber_pool* pool) { ++pool->total; if (pool->high_water < pool->total) pool->high_water = pool->total; } inline void decrement_pool_total(cilk_fiber_pool* pool, int fibers_freed) { pool->total -= fibers_freed; } /** * @brief Free fibers from this pool until we have at most @c * num_to_keep fibers remaining, and then put a fiber back. * * @pre We do not hold @c pool->lock * @post After completion, we do not hold @c pool->lock */ static void cilk_fiber_pool_free_fibers_from_pool(cilk_fiber_pool* pool, unsigned num_to_keep, cilk_fiber* fiber_to_return) { // Free our own fibers, until we fall below our desired threshold. // Each iteration of this loop proceeds in the following stages: // 1. Acquire the pool lock, // 2. Grabs up to B fibers from the pool, stores them into a buffer. // 3. Check if pool is empty enough. If yes, put the last fiber back, // and remember that we should quit. // 4. Release the pool lock, and actually free any buffered fibers. // 5. Check if we are done and should exit the loop. Otherwise, try again. // const bool need_lock = pool->lock; bool last_fiber_returned = false; do { const int B = 10; // Pull at most this many fibers from the // parent for one lock acquisition. Make // this value large enough to amortize // against the cost of acquiring and // releasing the lock. int num_to_free = 0; cilk_fiber* fibers_to_free[B]; // Stage 1: Grab the lock. if (need_lock) { spin_mutex_lock(pool->lock); } // Stage 2: Grab up to B fibers to free. int fibers_freed = 0; while ((pool->size > num_to_keep) && (num_to_free < B)) { fibers_to_free[num_to_free++] = pool->fibers[--pool->size]; fibers_freed++; } decrement_pool_total(pool, fibers_freed); // Stage 3. Pool is below threshold. Put extra fiber back. if (pool->size <= num_to_keep) { // Put the last fiber back into the pool. if (fiber_to_return) { CILK_ASSERT(pool->size < pool->max_size); pool->fibers[pool->size] = fiber_to_return; pool->size++; } last_fiber_returned = true; } // Stage 4: Release the lock, and actually free any fibers // buffered. if (need_lock) { spin_mutex_unlock(pool->lock); } for (int i = 0; i < num_to_free; ++i) { fibers_to_free[i]->deallocate_to_heap(); } } while (!last_fiber_returned); } /****************************************************************** * TBD: We want to simplify / rework the logic for allocating and * deallocating fibers, so that they are hopefully simpler and work * more elegantly for more than two levels. ******************************************************************/ /** * @brief Transfer fibers from @c pool to @c pool->parent. * * @pre Must hold @c pool->lock if it exists. * @post After completion, some number of fibers * have been moved from this pool to the parent. * The lock @c pool->lock is still held. * * TBD: Do we wish to guarantee that the lock has never been * released? It may depend on the implementation... */ static void cilk_fiber_pool_move_fibers_to_parent_pool(cilk_fiber_pool* pool, unsigned num_to_keep) { // ASSERT: We should hold the lock on pool (if it has one). CILK_ASSERT(pool->parent); cilk_fiber_pool* parent_pool = pool->parent; // Move fibers from our pool to the parent until we either run out // of space in the parent, or hit our threshold. // // This operation must be done while holding the parent lock. // If the parent pool appears to be full, just return early. if (parent_pool->size >= parent_pool->max_size) return; spin_mutex_lock(pool->parent->lock); while ((parent_pool->size < parent_pool->max_size) && (pool->size > num_to_keep)) { parent_pool->fibers[parent_pool->size++] = pool->fibers[--pool->size]; } // If the child pool has deallocated more than fibers to the heap // than it has allocated, then transfer this "surplus" to the // parent, so that the parent is free to allocate more from the // heap. // // This transfer means that the total in the parent can // temporarily go negative. if (pool->total < 0) { // Reduce parent total by the surplus we have in the local // pool. parent_pool->total += pool->total; pool->total = 0; } spin_mutex_unlock(pool->parent->lock); } void cilk_fiber_pool_init(cilk_fiber_pool* pool, cilk_fiber_pool* parent, size_t stack_size, unsigned buffer_size, int alloc_max, int is_shared) { #if FIBER_DEBUG >= 1 fprintf(stderr, "fiber_pool_init, pool=%p, parent=%p, alloc_max=%u\n", pool, parent, alloc_max); #endif pool->lock = (is_shared ? spin_mutex_create() : NULL); pool->parent = parent; pool->stack_size = stack_size; pool->max_size = buffer_size; pool->size = 0; pool->total = 0; pool->high_water = 0; pool->alloc_max = alloc_max; pool->fibers = (cilk_fiber**) __cilkrts_malloc(buffer_size * sizeof(cilk_fiber*)); CILK_ASSERT(NULL != pool->fibers); #ifdef __MIC__ #define PREALLOCATE_FIBERS #endif #ifdef PREALLOCATE_FIBERS // Pre-allocate 1/4 of fibers in the pools ahead of time. This // value is somewhat arbitrary. It was chosen to be less than the // threshold (of about 3/4) of fibers to keep in the pool when // transferring fibers to the parent. int pre_allocate_count = buffer_size/4; for (pool->size = 0; pool->size < pre_allocate_count; pool->size++) { pool->fibers[pool->size] = cilk_fiber::allocate_from_heap(pool->stack_size); } #endif } void cilk_fiber_pool_set_fiber_limit(cilk_fiber_pool* root_pool, unsigned max_fibers_to_allocate) { // Should only set limit on root pool, not children. CILK_ASSERT(NULL == root_pool->parent); root_pool->alloc_max = max_fibers_to_allocate; } void cilk_fiber_pool_destroy(cilk_fiber_pool* pool) { CILK_ASSERT(cilk_fiber_pool_sanity_check(pool, "pool_destroy")); // Lock my own pool, if I need to. if (pool->lock) { spin_mutex_lock(pool->lock); } // Give any remaining fibers to parent pool. if (pool->parent) { cilk_fiber_pool_move_fibers_to_parent_pool(pool, 0); } // Unlock pool. if (pool->lock) { spin_mutex_unlock(pool->lock); } // If I have any left in my pool, just free them myself. // This method may acquire the pool lock. cilk_fiber_pool_free_fibers_from_pool(pool, 0, NULL); // Destroy the lock if there is one. if (pool->lock) { spin_mutex_destroy(pool->lock); } __cilkrts_free(pool->fibers); } cilk_fiber* cilk_fiber_allocate(cilk_fiber_pool* pool) { CILK_ASSERT(cilk_fiber_pool_sanity_check(pool, "allocate")); return cilk_fiber::allocate(pool); } cilk_fiber* cilk_fiber_allocate_from_heap(size_t stack_size) { return cilk_fiber::allocate_from_heap(stack_size); } void cilk_fiber_reset_state(cilk_fiber* fiber, cilk_fiber_proc start_proc) { fiber->reset_state(start_proc); } int cilk_fiber_remove_reference(cilk_fiber *fiber, cilk_fiber_pool *pool) { return fiber->remove_reference(pool); } cilk_fiber* cilk_fiber_allocate_from_thread() { return cilk_fiber::allocate_from_thread(); } int cilk_fiber_deallocate_from_thread(cilk_fiber *fiber) { return fiber->deallocate_from_thread(); } int cilk_fiber_remove_reference_from_thread(cilk_fiber *fiber) { return fiber->remove_reference_from_thread(); } int cilk_fiber_is_allocated_from_thread(cilk_fiber *fiber) { return fiber->is_allocated_from_thread(); } #if SUPPORT_GET_CURRENT_FIBER cilk_fiber* cilk_fiber_get_current_fiber(void) { return cilk_fiber::get_current_fiber(); } #endif void cilk_fiber_suspend_self_and_resume_other(cilk_fiber* self, cilk_fiber* other) { self->suspend_self_and_resume_other(other); } void cilk_fiber::reset_state(cilk_fiber_proc start_proc) { // Setup the fiber and return. this->m_start_proc = start_proc; CILK_ASSERT(!this->is_resumable()); CILK_ASSERT(NULL == this->m_pending_remove_ref); CILK_ASSERT(NULL == this->m_pending_pool); } NORETURN cilk_fiber_remove_reference_from_self_and_resume_other(cilk_fiber* self, cilk_fiber_pool* self_pool, cilk_fiber* other) { #if FIBER_DEBUG >= 3 __cilkrts_worker* w = __cilkrts_get_tls_worker(); fprintf(stderr, "W=%d: cilk_fiber_deactivate_self_and_resume_other: self=%p, other=%p\n", w->self, self, other); #endif CILK_ASSERT(cilk_fiber_pool_sanity_check(self_pool, "remove_reference_from_self_resume_other")); self->remove_reference_from_self_and_resume_other(self_pool, other); // We should never return here. } void cilk_fiber_set_post_switch_proc(cilk_fiber *self, cilk_fiber_proc post_switch_proc) { self->set_post_switch_proc(post_switch_proc); } void cilk_fiber_invoke_tbb_stack_op(cilk_fiber* fiber, __cilk_tbb_stack_op op) { fiber->invoke_tbb_stack_op(op); } cilk_fiber_data* cilk_fiber_get_data(cilk_fiber* fiber) { return fiber->get_data(); /// TBD: Change this code to "return (cilk_fiber_data*)fiber;" // plus a static assert, so that this function is // more easily inlined by the compiler. } int cilk_fiber_is_resumable(cilk_fiber *fiber) { return fiber->is_resumable(); } char* cilk_fiber_get_stack_base(cilk_fiber *fiber) { return fiber->get_stack_base(); } #if defined(_WIN32) && 0 // Only works on Windows. Disable debugging for now. #define DBG_STACK_OPS(_fmt, ...) __cilkrts_dbgprintf(_fmt, __VA_ARGS__) #else #define DBG_STACK_OPS(_fmt, ...) #endif void cilk_fiber_set_stack_op(cilk_fiber *fiber, __cilk_tbb_stack_op_thunk o) { cilk_fiber_data *fdata = cilk_fiber_get_data(fiber); DBG_STACK_OPS ("cilk_fiber_set_stack_op - cilk_fiber %p, routine: %p, data: %p\n", fiber, o.routine, o.data); fdata->stack_op_routine = o.routine; fdata->stack_op_data = o.data; } #if 0 // Debugging function static const char *NameStackOp (enum __cilk_tbb_stack_op op) { switch(op) { case CILK_TBB_STACK_ORPHAN: return "CILK_TBB_STACK_ORPHAN"; case CILK_TBB_STACK_ADOPT: return "CILK_TBB_STACK_ADOPT"; case CILK_TBB_STACK_RELEASE: return "CILK_TBB_STACK_RELEASE"; default: return "Unknown"; } } #endif /* * Save TBB interop information for an unbound thread. It will get picked * up when the thread is bound to the runtime. */ void cilk_fiber_tbb_interop_save_stack_op_info(__cilk_tbb_stack_op_thunk o) { __cilk_tbb_stack_op_thunk *saved_thunk = __cilkrts_get_tls_tbb_interop(); DBG_STACK_OPS("Calling save_stack_op; o.routine=%p, o.data=%p, saved_thunk=%p\n", o.routine, o.data, saved_thunk); // If there is not already space allocated, allocate some. if (NULL == saved_thunk) { saved_thunk = (__cilk_tbb_stack_op_thunk*) __cilkrts_malloc(sizeof(__cilk_tbb_stack_op_thunk)); __cilkrts_set_tls_tbb_interop(saved_thunk); } *saved_thunk = o; DBG_STACK_OPS ("Unbound Thread %04x: tbb_interop_save_stack_op_info - saved info\n", cilkos_get_current_thread_id()); } /* * Save TBB interop information from the cilk_fiber. It will get picked * up when the thread is bound to the runtime next time. */ void cilk_fiber_tbb_interop_save_info_from_stack(cilk_fiber *fiber) { __cilk_tbb_stack_op_thunk *saved_thunk; cilk_fiber_data* fdata; if (NULL == fiber) return; fdata = cilk_fiber_get_data(fiber); // If there is no TBB interop data, just return if (NULL == fdata->stack_op_routine) return; saved_thunk = __cilkrts_get_tls_tbb_interop(); // If there is not already space allocated, allocate some. if (NULL == saved_thunk) { saved_thunk = (__cilk_tbb_stack_op_thunk*) __cilkrts_malloc(sizeof(__cilk_tbb_stack_op_thunk)); __cilkrts_set_tls_tbb_interop(saved_thunk); } saved_thunk->routine = fdata->stack_op_routine; saved_thunk->data = fdata->stack_op_data; } /* * If there's TBB interop information that was saved before the thread was * bound, apply it now */ void cilk_fiber_tbb_interop_use_saved_stack_op_info(cilk_fiber* fiber) { __cilk_tbb_stack_op_thunk *saved_thunk = __cilkrts_get_tls_tbb_interop(); CILK_ASSERT(fiber); // If we haven't allocated a TBB interop index, we don't have any saved info if (NULL == saved_thunk) { DBG_STACK_OPS ("cilk_fiber %p: tbb_interop_use_saved_stack_op_info - no saved info\n", fiber); return; } DBG_STACK_OPS ("cilk_fiber %p: tbb_interop_use_saved_stack_op_info - using saved info\n", fiber); // Associate the saved info with the __cilkrts_stack cilk_fiber_set_stack_op(fiber, *saved_thunk); // Free the saved data. We'll save it again if needed when the code // returns from the initial function cilk_fiber_tbb_interop_free_stack_op_info(); } /* * Free saved TBB interop memory. Should only be called when the thread is * not bound. */ void cilk_fiber_tbb_interop_free_stack_op_info(void) { __cilk_tbb_stack_op_thunk *saved_thunk = __cilkrts_get_tls_tbb_interop(); // If we haven't allocated a TBB interop index, we don't have any saved info if (NULL == saved_thunk) return; DBG_STACK_OPS ("tbb_interop_free_stack_op_info - freeing saved info\n"); // Free the memory and wipe out the TLS value __cilkrts_free(saved_thunk); __cilkrts_set_tls_tbb_interop(NULL); } #if NEED_FIBER_REF_COUNTS int cilk_fiber_has_references(cilk_fiber *fiber) { return (fiber->get_ref_count() > 0); } int cilk_fiber_get_ref_count(cilk_fiber *fiber) { return fiber->get_ref_count(); } void cilk_fiber_add_reference(cilk_fiber *fiber) { fiber->inc_ref_count(); } #endif // NEED_FIBER_REF_COUNTS } // End extern "C" cilk_fiber_sysdep* cilk_fiber::sysdep() { return static_cast(this); } cilk_fiber::cilk_fiber() : m_start_proc(NULL) , m_post_switch_proc(NULL) , m_pending_remove_ref(NULL) , m_pending_pool(NULL) , m_flags(0) { // Clear cilk_fiber_data base-class data members std::memset((cilk_fiber_data*) this, 0, sizeof(cilk_fiber_data)); // cilk_fiber data members init_ref_count(0); } cilk_fiber::cilk_fiber(std::size_t stack_size) { *this = cilk_fiber(); // A delegating constructor would be nice here this->stack_size = stack_size; } cilk_fiber::~cilk_fiber() { // Empty destructor. } char* cilk_fiber::get_stack_base() { return this->sysdep()->get_stack_base_sysdep(); } cilk_fiber* cilk_fiber::allocate_from_heap(std::size_t stack_size) { // Case 1: pool is NULL. create a new fiber from the heap // No need for locks here. cilk_fiber_sysdep* ret = (cilk_fiber_sysdep*) __cilkrts_malloc(sizeof(cilk_fiber_sysdep)); // Error condition. If we failed to allocate a fiber from the // heap, we are in trouble though... if (!ret) return NULL; ::new(ret) cilk_fiber_sysdep(stack_size); CILK_ASSERT(0 == ret->m_flags); CILK_ASSERT(NULL == ret->m_pending_remove_ref); CILK_ASSERT(NULL == ret->m_pending_pool); ret->init_ref_count(1); return ret; } #if USE_FIBER_TRY_ALLOCATE_FROM_POOL /** * Helper method: try to allocate a fiber from this pool or its * ancestors without going to the OS / heap. * * Returns allocated pool, or NULL if no pool is found. * * If pool contains a suitable fiber. Return it. Otherwise, try to * recursively grab a fiber from the parent pool, if there is one. * * This method will not allocate a fiber from the heap. * * This method could be written either recursively or iteratively. * It probably does not matter which one we do. * * @note This method is compiled, but may not be used unless the * USE_FIBER_TRY_ALLOCATE_FROM_POOL switch is set. */ cilk_fiber* cilk_fiber::try_allocate_from_pool_recursive(cilk_fiber_pool* pool) { cilk_fiber* ret = NULL; if (pool->size > 0) { // Try to get the lock. if (pool->lock) { // For some reason, it seems to be better to just block on the parent // pool lock, instead of using a try-lock? #define USE_TRY_LOCK_IN_FAST_ALLOCATE 0 #if USE_TRY_LOCK_IN_FAST_ALLOCATE int got_lock = spin_mutex_trylock(pool->lock); if (!got_lock) { // If we fail, skip to the parent. if (pool->parent) { return try_allocate_from_pool_recursive(pool->parent); } } #else spin_mutex_lock(pool->lock); #endif } // Check in the pool if we have the lock. if (pool->size > 0) { ret = pool->fibers[--pool->size]; } // Release the lock once we are done updating pool fields. if (pool->lock) { spin_mutex_unlock(pool->lock); } } if ((!ret) && (pool->parent)) { return try_allocate_from_pool_recursive(pool->parent); } if (ret) { // When we pull a fiber out of the pool, set its reference // count before we return it. ret->init_ref_count(1); } return ret; } #endif // USE_FIBER_TRY_ALLOCATE_FROM_POOL cilk_fiber* cilk_fiber::allocate(cilk_fiber_pool* pool) { // Pool should not be NULL in this method. But I'm not going to // actually assert it, because we are likely to seg fault anyway // if it is. // CILK_ASSERT(NULL != pool); cilk_fiber *ret = NULL; #if USE_FIBER_TRY_ALLOCATE_FROM_POOL // "Fast" path, which doesn't go to the heap or OS until checking // the ancestors first. ret = try_allocate_from_pool_recursive(pool); if (ret) return ret; #endif // If we don't get anything from the "fast path", then go through // a slower path to look for a fiber. // // 1. Lock the pool if it is shared. // 2. Look in our local pool. If we find one, release the lock // and quit searching. // 3. Otherwise, check whether we can allocate from heap. // 4. Release the lock if it was acquired. // 5. Try to allocate from the heap, if step 3 said we could. // If we find a fiber, then quit searching. // 6. If none of these steps work, just recursively try again // from the parent. // 1. Lock the pool if it is shared. if (pool->lock) { spin_mutex_lock(pool->lock); } // 2. Look in local pool. if (pool->size > 0) { ret = pool->fibers[--pool->size]; if (ret) { // If we found one, release the lock once we are // done updating pool fields, and break out of the // loop. if (pool->lock) { spin_mutex_unlock(pool->lock); } // When we pull a fiber out of the pool, set its reference // count just in case. ret->init_ref_count(1); return ret; } } // 3. Check whether we can allocate from the heap. bool can_allocate_from_heap = false; if (pool->total < pool->alloc_max) { // Track that we are allocating a new fiber from the // heap, originating from this pool. // This increment may be undone if we happen to fail to // allocate from the heap. increment_pool_total(pool); can_allocate_from_heap = true; } // 4. Unlock the pool, and then allocate from the heap. if (pool->lock) { spin_mutex_unlock(pool->lock); } // 5. Actually try to allocate from the heap / OS. if (can_allocate_from_heap) { ret = allocate_from_heap(pool->stack_size); // If we got something from the heap, just return it. if (ret) { return ret; } // Otherwise, we failed in our attempt to allocate a // fiber from the heap. Grab the lock and decrement // the total again. if (pool->lock) { spin_mutex_lock(pool->lock); } decrement_pool_total(pool, 1); if (pool->lock) { spin_mutex_unlock(pool->lock); } } // 6. If we get here, then searching this pool failed. Go search // the parent instead if we have one. if (pool->parent) { return allocate(pool->parent); } return ret; } int cilk_fiber::remove_reference(cilk_fiber_pool* pool) { int ref_count = this->dec_ref_count(); if (ref_count == 0) { if (pool) { deallocate_self(pool); } else { deallocate_to_heap(); } } return ref_count; } cilk_fiber* cilk_fiber::allocate_from_thread() { void* retmem = __cilkrts_malloc(sizeof(cilk_fiber_sysdep)); CILK_ASSERT(retmem); cilk_fiber_sysdep* ret = ::new(retmem) cilk_fiber_sysdep(from_thread); // A fiber allocated from a thread begins with a reference count // of 2. The first is for being created, and the second is for // being running. // // Suspending this fiber will decrement the count down to 1. ret->init_ref_count(2); #if SUPPORT_GET_CURRENT_FIBER // We're creating the main fiber for this thread. Set this fiber as the // current fiber. cilkos_set_tls_cilk_fiber(ret); #endif return ret; } int cilk_fiber::deallocate_from_thread() { CILK_ASSERT(this->is_allocated_from_thread()); #if SUPPORT_GET_CURRENT_FIBER CILK_ASSERT(this == cilkos_get_tls_cilk_fiber()); // Reverse of "allocate_from_thread". cilkos_set_tls_cilk_fiber(NULL); #endif this->assert_ref_count_at_least(2); // Suspending the fiber should conceptually decrement the ref // count by 1. cilk_fiber_sysdep* self = this->sysdep(); self->convert_fiber_back_to_thread(); // Then, freeing the fiber itself decrements the ref count again. int ref_count = this->sub_from_ref_count(2); if (ref_count == 0) { self->~cilk_fiber_sysdep(); __cilkrts_free(self); } return ref_count; } int cilk_fiber::remove_reference_from_thread() { int ref_count = dec_ref_count(); if (ref_count == 0) { cilk_fiber_sysdep* self = this->sysdep(); self->~cilk_fiber_sysdep(); __cilkrts_free(self); } return ref_count; } #if SUPPORT_GET_CURRENT_FIBER cilk_fiber* cilk_fiber::get_current_fiber() { return cilk_fiber_sysdep::get_current_fiber_sysdep(); } #endif void cilk_fiber::do_post_switch_actions() { if (m_post_switch_proc) { cilk_fiber_proc proc = m_post_switch_proc; m_post_switch_proc = NULL; proc(this); } if (m_pending_remove_ref) { m_pending_remove_ref->remove_reference(m_pending_pool); // Even if we don't free it, m_pending_remove_ref = NULL; m_pending_pool = NULL; } } void cilk_fiber::suspend_self_and_resume_other(cilk_fiber* other) { #if FIBER_DEBUG >=1 fprintf(stderr, "suspend_self_and_resume_other: self =%p, other=%p [owner=%p, resume_sf=%p]\n", this, other, other->owner, other->resume_sf); #endif // Decrement my reference count (to suspend) // Increment other's count (to resume) // Suspended fiber should have a reference count of at least 1. (It is not in a pool). this->dec_ref_count(); other->inc_ref_count(); this->assert_ref_count_at_least(1); // Pass along my owner. other->owner = this->owner; this->owner = NULL; // Change this fiber to resumable. CILK_ASSERT(!this->is_resumable()); this->set_resumable(true); // Normally, I'd assert other->is_resumable(). But this flag may // be false the first time we try to "resume" a fiber. cilk_fiber_sysdep* self = this->sysdep(); self->suspend_self_and_resume_other_sysdep(other->sysdep()); // HAVE RESUMED EXECUTION // When we come back here, we should have at least two references: // one for the fiber being allocated / out of a pool, and one for it being active. this->assert_ref_count_at_least(2); } NORETURN cilk_fiber::remove_reference_from_self_and_resume_other(cilk_fiber_pool* self_pool, cilk_fiber* other) { // Decrement my reference count once (to suspend) // Increment other's count (to resume) // Suspended fiber should have a reference count of at least 1. (It is not in a pool). this->dec_ref_count(); other->inc_ref_count(); // Set a pending remove reference for this fiber, once we have // actually switched off. other->m_pending_remove_ref = this; other->m_pending_pool = self_pool; // Pass along my owner. other->owner = this->owner; this->owner = NULL; // Since we are deallocating self, this fiber does not become // resumable. CILK_ASSERT(!this->is_resumable()); cilk_fiber_sysdep* self = this->sysdep(); self->jump_to_resume_other_sysdep(other->sysdep()); __cilkrts_bug("Deallocating fiber. We should never come back here."); std::abort(); } void cilk_fiber::deallocate_to_heap() { cilk_fiber_sysdep* self = this->sysdep(); self->~cilk_fiber_sysdep(); __cilkrts_free(self); } void cilk_fiber::deallocate_self(cilk_fiber_pool* pool) { this->set_resumable(false); CILK_ASSERT(NULL != pool); CILK_ASSERT(!this->is_allocated_from_thread()); this->assert_ref_count_equals(0); // Cases: // // 1. pool has space: Add to this pool. // 2. pool is full: Give some fibers to parent, and then free // enough to make space for the fiber we are deallocating. // Then put the fiber back into the pool. const bool need_lock = pool->lock; // Grab the lock for the remaining cases. if (need_lock) { spin_mutex_lock(pool->lock); } // Case 1: this pool has space. Return the fiber. if (pool->size < pool->max_size) { // Add this fiber to pool pool->fibers[pool->size++] = this; if (need_lock) { spin_mutex_unlock(pool->lock); } return; } // Case 2: Pool is full. // // First free up some space by giving fibers to the parent. if (pool->parent) { // Pool is full. Move all but "num_to_keep" fibers to parent, // if we can. unsigned num_to_keep = pool->max_size/2 + pool->max_size/4; cilk_fiber_pool_move_fibers_to_parent_pool(pool, num_to_keep); } if (need_lock) { spin_mutex_unlock(pool->lock); } // Now, free a fiber to make room for the one we need to put back, // and then put this fiber back. This step may actually return // fibers to the heap. cilk_fiber_pool_free_fibers_from_pool(pool, pool->max_size -1, this); } // NOTE: Except for print-debug, this code is the same as in Windows. void cilk_fiber::invoke_tbb_stack_op(__cilk_tbb_stack_op op) { cilk_fiber_data *fdata = this->get_data(); if (0 == fdata->stack_op_routine) { if (CILK_TBB_STACK_RELEASE != op) DBG_STACK_OPS ("Wkr %p: invoke_tbb_stack_op - %s (%d) for cilk_fiber %p, fiber %p, thread id %04x - No stack op routine\n", fdata->owner, NameStackOp(op), op, fdata, this, cilkos_get_current_thread_id()); return; } // Call TBB to do it's thing DBG_STACK_OPS ("Wkr %p: invoke_tbb_stack_op - op %s data %p for cilk_fiber %p, fiber %p, thread id %04x\n", fdata->owner, NameStackOp(op), fdata->stack_op_data, fdata, this, cilkos_get_current_thread_id()); (*fdata->stack_op_routine)(op, fdata->stack_op_data); if (op == CILK_TBB_STACK_RELEASE) { fdata->stack_op_routine = 0; fdata->stack_op_data = 0; } } #if NEED_FIBER_REF_COUNTS void cilk_fiber::atomic_inc_ref_count() { cilkos_atomic_add(&m_outstanding_references, 1); } long cilk_fiber::atomic_dec_ref_count() { return cilkos_atomic_add(&m_outstanding_references, -1); } long cilk_fiber::atomic_sub_from_ref_count(long v) { return cilkos_atomic_add(&m_outstanding_references, -v); } #endif // NEED_FIBER_REF_COUNTS /* End cilk_fibers.cpp */