//===-- tsan_mman.cc ------------------------------------------------------===// // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file is a part of ThreadSanitizer (TSan), a race detector. // //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "tsan_mman.h" #include "tsan_rtl.h" #include "tsan_report.h" #include "tsan_flags.h" // May be overriden by front-end. extern "C" void WEAK __tsan_malloc_hook(void *ptr, uptr size) { (void)ptr; (void)size; } extern "C" void WEAK __tsan_free_hook(void *ptr) { (void)ptr; } namespace __tsan { COMPILER_CHECK(sizeof(MBlock) == 16); void MBlock::Lock() { atomic_uintptr_t *a = reinterpret_cast(this); uptr v = atomic_load(a, memory_order_relaxed); for (int iter = 0;; iter++) { if (v & 1) { if (iter < 10) proc_yield(20); else internal_sched_yield(); v = atomic_load(a, memory_order_relaxed); continue; } if (atomic_compare_exchange_weak(a, &v, v | 1, memory_order_acquire)) break; } } void MBlock::Unlock() { atomic_uintptr_t *a = reinterpret_cast(this); uptr v = atomic_load(a, memory_order_relaxed); DCHECK(v & 1); atomic_store(a, v & ~1, memory_order_relaxed); } struct MapUnmapCallback { void OnMap(uptr p, uptr size) const { } void OnUnmap(uptr p, uptr size) const { // We are about to unmap a chunk of user memory. // Mark the corresponding shadow memory as not needed. DontNeedShadowFor(p, size); } }; static char allocator_placeholder[sizeof(Allocator)] ALIGNED(64); Allocator *allocator() { return reinterpret_cast(&allocator_placeholder); } void InitializeAllocator() { allocator()->Init(); } void AllocatorThreadStart(ThreadState *thr) { allocator()->InitCache(&thr->alloc_cache); internal_allocator()->InitCache(&thr->internal_alloc_cache); } void AllocatorThreadFinish(ThreadState *thr) { allocator()->DestroyCache(&thr->alloc_cache); internal_allocator()->DestroyCache(&thr->internal_alloc_cache); } void AllocatorPrintStats() { allocator()->PrintStats(); } static void SignalUnsafeCall(ThreadState *thr, uptr pc) { if (!thr->in_signal_handler || !flags()->report_signal_unsafe) return; Context *ctx = CTX(); StackTrace stack; stack.ObtainCurrent(thr, pc); ThreadRegistryLock l(ctx->thread_registry); ScopedReport rep(ReportTypeSignalUnsafe); if (!IsFiredSuppression(ctx, rep, stack)) { rep.AddStack(&stack); OutputReport(ctx, rep, rep.GetReport()->stacks[0]); } } void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align) { CHECK_GT(thr->in_rtl, 0); if ((sz >= (1ull << 40)) || (align >= (1ull << 40))) return AllocatorReturnNull(); void *p = allocator()->Allocate(&thr->alloc_cache, sz, align); if (p == 0) return 0; MBlock *b = new(allocator()->GetMetaData(p)) MBlock; b->Init(sz, thr->tid, CurrentStackId(thr, pc)); if (CTX() && CTX()->initialized) { if (thr->ignore_reads_and_writes == 0) MemoryRangeImitateWrite(thr, pc, (uptr)p, sz); else MemoryResetRange(thr, pc, (uptr)p, sz); } DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p); SignalUnsafeCall(thr, pc); return p; } void user_free(ThreadState *thr, uptr pc, void *p) { CHECK_GT(thr->in_rtl, 0); CHECK_NE(p, (void*)0); DPrintf("#%d: free(%p)\n", thr->tid, p); MBlock *b = (MBlock*)allocator()->GetMetaData(p); if (b->ListHead()) { MBlock::ScopedLock l(b); for (SyncVar *s = b->ListHead(); s;) { SyncVar *res = s; s = s->next; StatInc(thr, StatSyncDestroyed); res->mtx.Lock(); res->mtx.Unlock(); DestroyAndFree(res); } b->ListReset(); } if (CTX() && CTX()->initialized && thr->in_rtl == 1) { if (thr->ignore_reads_and_writes == 0) MemoryRangeFreed(thr, pc, (uptr)p, b->Size()); } allocator()->Deallocate(&thr->alloc_cache, p); SignalUnsafeCall(thr, pc); } void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) { CHECK_GT(thr->in_rtl, 0); void *p2 = 0; // FIXME: Handle "shrinking" more efficiently, // it seems that some software actually does this. if (sz) { p2 = user_alloc(thr, pc, sz); if (p2 == 0) return 0; if (p) { MBlock *b = user_mblock(thr, p); CHECK_NE(b, 0); internal_memcpy(p2, p, min(b->Size(), sz)); } } if (p) user_free(thr, pc, p); return p2; } uptr user_alloc_usable_size(ThreadState *thr, uptr pc, void *p) { CHECK_GT(thr->in_rtl, 0); if (p == 0) return 0; MBlock *b = (MBlock*)allocator()->GetMetaData(p); return b ? b->Size() : 0; } MBlock *user_mblock(ThreadState *thr, void *p) { CHECK_NE(p, 0); Allocator *a = allocator(); void *b = a->GetBlockBegin(p); if (b == 0) return 0; return (MBlock*)a->GetMetaData(b); } void invoke_malloc_hook(void *ptr, uptr size) { Context *ctx = CTX(); ThreadState *thr = cur_thread(); if (ctx == 0 || !ctx->initialized || thr->in_rtl) return; __tsan_malloc_hook(ptr, size); } void invoke_free_hook(void *ptr) { Context *ctx = CTX(); ThreadState *thr = cur_thread(); if (ctx == 0 || !ctx->initialized || thr->in_rtl) return; __tsan_free_hook(ptr); } void *internal_alloc(MBlockType typ, uptr sz) { ThreadState *thr = cur_thread(); CHECK_GT(thr->in_rtl, 0); CHECK_LE(sz, InternalSizeClassMap::kMaxSize); if (thr->nomalloc) { thr->nomalloc = 0; // CHECK calls internal_malloc(). CHECK(0); } return InternalAlloc(sz, &thr->internal_alloc_cache); } void internal_free(void *p) { ThreadState *thr = cur_thread(); CHECK_GT(thr->in_rtl, 0); if (thr->nomalloc) { thr->nomalloc = 0; // CHECK calls internal_malloc(). CHECK(0); } InternalFree(p, &thr->internal_alloc_cache); } } // namespace __tsan using namespace __tsan; extern "C" { uptr __tsan_get_current_allocated_bytes() { u64 stats[AllocatorStatCount]; allocator()->GetStats(stats); u64 m = stats[AllocatorStatMalloced]; u64 f = stats[AllocatorStatFreed]; return m >= f ? m - f : 1; } uptr __tsan_get_heap_size() { u64 stats[AllocatorStatCount]; allocator()->GetStats(stats); u64 m = stats[AllocatorStatMmapped]; u64 f = stats[AllocatorStatUnmapped]; return m >= f ? m - f : 1; } uptr __tsan_get_free_bytes() { return 1; } uptr __tsan_get_unmapped_bytes() { return 1; } uptr __tsan_get_estimated_allocated_size(uptr size) { return size; } bool __tsan_get_ownership(void *p) { return allocator()->GetBlockBegin(p) != 0; } uptr __tsan_get_allocated_size(void *p) { if (p == 0) return 0; p = allocator()->GetBlockBegin(p); if (p == 0) return 0; MBlock *b = (MBlock*)allocator()->GetMetaData(p); return b->Size(); } void __tsan_on_thread_idle() { ThreadState *thr = cur_thread(); allocator()->SwallowCache(&thr->alloc_cache); internal_allocator()->SwallowCache(&thr->internal_alloc_cache); } } // extern "C"