diff options
| -rw-r--r-- | runtime/arch/arm/quick_entrypoints_arm.S | 2 | ||||
| -rw-r--r-- | runtime/arch/arm64/quick_entrypoints_arm64.S | 77 | ||||
| -rw-r--r-- | runtime/arch/stub_test.cc | 129 | ||||
| -rw-r--r-- | runtime/arch/x86_64/quick_entrypoints_x86_64.S | 4 |
4 files changed, 163 insertions, 49 deletions
diff --git a/runtime/arch/arm/quick_entrypoints_arm.S b/runtime/arch/arm/quick_entrypoints_arm.S index dcf4561a9..488656176 100644 --- a/runtime/arch/arm/quick_entrypoints_arm.S +++ b/runtime/arch/arm/quick_entrypoints_arm.S @@ -442,8 +442,8 @@ ENTRY art_quick_unlock_object cmp r1, #65536 bpl .Lrecursive_thin_unlock @ transition to unlocked, r3 holds 0 - str r3, [r0, #LOCK_WORD_OFFSET] dmb ish @ full (StoreLoad) memory barrier + str r3, [r0, #LOCK_WORD_OFFSET] bx lr .Lrecursive_thin_unlock: sub r1, r1, #65536 diff --git a/runtime/arch/arm64/quick_entrypoints_arm64.S b/runtime/arch/arm64/quick_entrypoints_arm64.S index f7cb25408..c056b2f81 100644 --- a/runtime/arch/arm64/quick_entrypoints_arm64.S +++ b/runtime/arch/arm64/quick_entrypoints_arm64.S @@ -998,8 +998,81 @@ ENTRY art_quick_handle_fill_data DELIVER_PENDING_EXCEPTION END art_quick_handle_fill_data -UNIMPLEMENTED art_quick_lock_object -UNIMPLEMENTED art_quick_unlock_object + /* + * Entry from managed code that calls artLockObjectFromCode, may block for GC. x0 holds the + * possibly null object to lock. + * + * Derived from arm32 code. + */ + .extern artLockObjectFromCode +ENTRY art_quick_lock_object + cbz w0, .Lslow_lock + add x4, x0, #LOCK_WORD_OFFSET // exclusive load/store had no immediate anymore +.Lretry_lock: + ldr w2, [xSELF, #THREAD_ID_OFFSET] // TODO: Can the thread ID really change during the loop? + ldxr w1, [x4] + cbnz w1, .Lnot_unlocked // already thin locked + stxr w3, w2, [x4] + cbnz w3, .Lstrex_fail // store failed, retry + dmb ishld // full (LoadLoad) memory barrier, TODO: acquire-release + ret +.Lstrex_fail: + b .Lretry_lock // unlikely forward branch, need to reload and recheck r1/r2 +.Lnot_unlocked: + lsr w3, w1, 30 + cbnz w3, .Lslow_lock // if either of the top two bits are set, go slow path + eor w2, w1, w2 // lock_word.ThreadId() ^ self->ThreadId() + uxth w2, w2 // zero top 16 bits + cbnz w2, .Lslow_lock // lock word and self thread id's match -> recursive lock + // else contention, go to slow path + add w2, w1, #65536 // increment count in lock word placing in w2 for storing + lsr w1, w2, 30 // if either of the top two bits are set, we overflowed. + cbnz w1, .Lslow_lock // if we overflow the count go slow path + str w2, [x0, #LOCK_WORD_OFFSET]// no need for stxr as we hold the lock + ret +.Lslow_lock: + SETUP_REF_ONLY_CALLEE_SAVE_FRAME // save callee saves in case we block + mov x1, xSELF // pass Thread::Current + mov x2, sp // pass SP + bl artLockObjectFromCode // (Object* obj, Thread*, SP) + RESTORE_REF_ONLY_CALLEE_SAVE_FRAME + RETURN_IF_W0_IS_ZERO_OR_DELIVER +END art_quick_lock_object + + /* + * Entry from managed code that calls artUnlockObjectFromCode and delivers exception on failure. + * x0 holds the possibly null object to lock. + * + * Derived from arm32 code. + */ + .extern artUnlockObjectFromCode +ENTRY art_quick_unlock_object + cbz x0, .Lslow_unlock + ldr w1, [x0, #LOCK_WORD_OFFSET] + lsr w2, w1, 30 + cbnz w2, .Lslow_unlock // if either of the top two bits are set, go slow path + ldr w2, [xSELF, #THREAD_ID_OFFSET] + eor w3, w1, w2 // lock_word.ThreadId() ^ self->ThreadId() + uxth w3, w3 // zero top 16 bits + cbnz w3, .Lslow_unlock // do lock word and self thread id's match? + cmp w1, #65536 + bpl .Lrecursive_thin_unlock + // transition to unlocked, w3 holds 0 + dmb ish // full (StoreLoad) memory barrier + str w3, [x0, #LOCK_WORD_OFFSET] + ret +.Lrecursive_thin_unlock: + sub w1, w1, #65536 + str w1, [x0, #LOCK_WORD_OFFSET] + ret +.Lslow_unlock: + SETUP_REF_ONLY_CALLEE_SAVE_FRAME // save callee saves in case exception allocation triggers GC + mov x1, xSELF // pass Thread::Current + mov x2, sp // pass SP + bl artUnlockObjectFromCode // (Object* obj, Thread*, SP) + RESTORE_REF_ONLY_CALLEE_SAVE_FRAME + RETURN_IF_W0_IS_ZERO_OR_DELIVER +END art_quick_unlock_object /* * Entry from managed code that calls artIsAssignableFromCode and on failure calls diff --git a/runtime/arch/stub_test.cc b/runtime/arch/stub_test.cc index 86f52aa46..76d028db7 100644 --- a/runtime/arch/stub_test.cc +++ b/runtime/arch/stub_test.cc @@ -56,6 +56,7 @@ class StubTest : public CommonRuntimeTest { return &self->tlsPtr_; } + public: size_t Invoke3(size_t arg0, size_t arg1, size_t arg2, uintptr_t code, Thread* self) { // Push a transition back into managed code onto the linked list in thread. ManagedStack fragment; @@ -169,7 +170,6 @@ class StubTest : public CommonRuntimeTest { return result; } - public: // TODO: Set up a frame according to referrer's specs. size_t Invoke3WithReferrer(size_t arg0, size_t arg1, size_t arg2, uintptr_t code, Thread* self, mirror::ArtMethod* referrer) { @@ -357,12 +357,12 @@ TEST_F(StubTest, Memcpy) { #endif } -#if defined(__i386__) || defined(__arm__) || defined(__x86_64__) +#if defined(__i386__) || defined(__arm__) || defined(__aarch64__) || defined(__x86_64__) extern "C" void art_quick_lock_object(void); #endif TEST_F(StubTest, LockObject) { -#if defined(__i386__) || defined(__arm__) || defined(__x86_64__) +#if defined(__i386__) || defined(__arm__) || defined(__aarch64__) || defined(__x86_64__) static constexpr size_t kThinLockLoops = 100; Thread* self = Thread::Current(); @@ -396,8 +396,21 @@ TEST_F(StubTest, LockObject) { EXPECT_EQ(l_inc.ThinLockCount(), i); } - // TODO: Improve this test. Somehow force it to go to fat locked. But that needs another thread. + // Force a fat lock by running identity hashcode to fill up lock word. + SirtRef<mirror::Object> obj2(soa.Self(), mirror::String::AllocFromModifiedUtf8(soa.Self(), + "hello, world!")); + + obj2->IdentityHashCode(); + + Invoke3(reinterpret_cast<size_t>(obj2.get()), 0U, 0U, + reinterpret_cast<uintptr_t>(&art_quick_lock_object), self); + + LockWord lock_after2 = obj2->GetLockWord(false); + LockWord::LockState new_state2 = lock_after2.GetState(); + EXPECT_EQ(LockWord::LockState::kFatLocked, new_state2); + EXPECT_NE(lock_after2.FatLockMonitor(), static_cast<Monitor*>(nullptr)); + // Test done. #else LOG(INFO) << "Skipping lock_object as I don't know how to do that on " << kRuntimeISA; // Force-print to std::cout so it's also outside the logcat. @@ -419,13 +432,14 @@ class RandGen { }; -#if defined(__i386__) || defined(__arm__) || defined(__x86_64__) +#if defined(__i386__) || defined(__arm__) || defined(__aarch64__) || defined(__x86_64__) extern "C" void art_quick_lock_object(void); extern "C" void art_quick_unlock_object(void); #endif -TEST_F(StubTest, UnlockObject) { -#if defined(__i386__) || defined(__arm__) || defined(__x86_64__) +// NO_THREAD_SAFETY_ANALYSIS as we do not want to grab exclusive mutator lock for MonitorInfo. +static void TestUnlockObject(StubTest* test) NO_THREAD_SAFETY_ANALYSIS { +#if defined(__i386__) || defined(__arm__) || defined(__aarch64__) || defined(__x86_64__) static constexpr size_t kThinLockLoops = 100; Thread* self = Thread::Current(); @@ -439,8 +453,8 @@ TEST_F(StubTest, UnlockObject) { LockWord::LockState old_state = lock.GetState(); EXPECT_EQ(LockWord::LockState::kUnlocked, old_state); - Invoke3(reinterpret_cast<size_t>(obj.get()), 0U, 0U, - reinterpret_cast<uintptr_t>(&art_quick_unlock_object), self); + test->Invoke3(reinterpret_cast<size_t>(obj.get()), 0U, 0U, + reinterpret_cast<uintptr_t>(&art_quick_unlock_object), self); // This should be an illegal monitor state. EXPECT_TRUE(self->IsExceptionPending()); @@ -450,15 +464,15 @@ TEST_F(StubTest, UnlockObject) { LockWord::LockState new_state = lock_after.GetState(); EXPECT_EQ(LockWord::LockState::kUnlocked, new_state); - Invoke3(reinterpret_cast<size_t>(obj.get()), 0U, 0U, - reinterpret_cast<uintptr_t>(&art_quick_lock_object), self); + test->Invoke3(reinterpret_cast<size_t>(obj.get()), 0U, 0U, + reinterpret_cast<uintptr_t>(&art_quick_lock_object), self); LockWord lock_after2 = obj->GetLockWord(false); LockWord::LockState new_state2 = lock_after2.GetState(); EXPECT_EQ(LockWord::LockState::kThinLocked, new_state2); - Invoke3(reinterpret_cast<size_t>(obj.get()), 0U, 0U, - reinterpret_cast<uintptr_t>(&art_quick_unlock_object), self); + test->Invoke3(reinterpret_cast<size_t>(obj.get()), 0U, 0U, + reinterpret_cast<uintptr_t>(&art_quick_unlock_object), self); LockWord lock_after3 = obj->GetLockWord(false); LockWord::LockState new_state3 = lock_after3.GetState(); @@ -472,13 +486,16 @@ TEST_F(StubTest, UnlockObject) { constexpr size_t kNumberOfLocks = 10; // Number of objects = lock constexpr size_t kIterations = 10000; // Number of iterations + constexpr size_t kMoveToFat = 1000; // Chance of 1:kMoveFat to make a lock fat. size_t counts[kNumberOfLocks]; + bool fat[kNumberOfLocks]; // Whether a lock should be thin or fat. SirtRef<mirror::String>* objects[kNumberOfLocks]; // Initialize = allocate. for (size_t i = 0; i < kNumberOfLocks; ++i) { counts[i] = 0; + fat[i] = false; objects[i] = new SirtRef<mirror::String>(soa.Self(), mirror::String::AllocFromModifiedUtf8(soa.Self(), "")); } @@ -487,36 +504,57 @@ TEST_F(StubTest, UnlockObject) { // Select which lock to update. size_t index = r.next() % kNumberOfLocks; - bool lock; // Whether to lock or unlock in this step. - if (counts[index] == 0) { - lock = true; - } else if (counts[index] == kThinLockLoops) { - lock = false; - } else { - // Randomly. - lock = r.next() % 2 == 0; - } + // Make lock fat? + if (!fat[index] && (r.next() % kMoveToFat == 0)) { + fat[index] = true; + objects[index]->get()->IdentityHashCode(); - if (lock) { - Invoke3(reinterpret_cast<size_t>(objects[index]->get()), 0U, 0U, - reinterpret_cast<uintptr_t>(&art_quick_lock_object), self); - counts[index]++; + LockWord lock_iter = objects[index]->get()->GetLockWord(false); + LockWord::LockState iter_state = lock_iter.GetState(); + if (counts[index] == 0) { + EXPECT_EQ(LockWord::LockState::kHashCode, iter_state); + } else { + EXPECT_EQ(LockWord::LockState::kFatLocked, iter_state); + } } else { - Invoke3(reinterpret_cast<size_t>(objects[index]->get()), 0U, 0U, - reinterpret_cast<uintptr_t>(&art_quick_unlock_object), self); - counts[index]--; - } + bool lock; // Whether to lock or unlock in this step. + if (counts[index] == 0) { + lock = true; + } else if (counts[index] == kThinLockLoops) { + lock = false; + } else { + // Randomly. + lock = r.next() % 2 == 0; + } - EXPECT_FALSE(self->IsExceptionPending()); + if (lock) { + test-> Invoke3(reinterpret_cast<size_t>(objects[index]->get()), 0U, 0U, + reinterpret_cast<uintptr_t>(&art_quick_lock_object), self); + counts[index]++; + } else { + test->Invoke3(reinterpret_cast<size_t>(objects[index]->get()), 0U, 0U, + reinterpret_cast<uintptr_t>(&art_quick_unlock_object), self); + counts[index]--; + } - // Check the new state. - LockWord lock_iter = objects[index]->get()->GetLockWord(false); - LockWord::LockState iter_state = lock_iter.GetState(); - if (counts[index] > 0) { - EXPECT_EQ(LockWord::LockState::kThinLocked, iter_state); - EXPECT_EQ(counts[index] - 1, lock_iter.ThinLockCount()); - } else { - EXPECT_EQ(LockWord::LockState::kUnlocked, iter_state); + EXPECT_FALSE(self->IsExceptionPending()); + + // Check the new state. + LockWord lock_iter = objects[index]->get()->GetLockWord(true); + LockWord::LockState iter_state = lock_iter.GetState(); + if (fat[index]) { + // Abuse MonitorInfo. + EXPECT_EQ(LockWord::LockState::kFatLocked, iter_state) << index; + MonitorInfo info(objects[index]->get()); + EXPECT_EQ(counts[index], info.entry_count_) << index; + } else { + if (counts[index] > 0) { + EXPECT_EQ(LockWord::LockState::kThinLocked, iter_state); + EXPECT_EQ(counts[index] - 1, lock_iter.ThinLockCount()); + } else { + EXPECT_EQ(LockWord::LockState::kUnlocked, iter_state); + } + } } } @@ -526,21 +564,21 @@ TEST_F(StubTest, UnlockObject) { size_t index = kNumberOfLocks - 1 - i; size_t count = counts[index]; while (count > 0) { - Invoke3(reinterpret_cast<size_t>(objects[index]->get()), 0U, 0U, - reinterpret_cast<uintptr_t>(&art_quick_unlock_object), self); + test->Invoke3(reinterpret_cast<size_t>(objects[index]->get()), 0U, 0U, + reinterpret_cast<uintptr_t>(&art_quick_unlock_object), self); count--; } LockWord lock_after4 = objects[index]->get()->GetLockWord(false); LockWord::LockState new_state4 = lock_after4.GetState(); - EXPECT_EQ(LockWord::LockState::kUnlocked, new_state4); + EXPECT_TRUE(LockWord::LockState::kUnlocked == new_state4 + || LockWord::LockState::kFatLocked == new_state4); delete objects[index]; } - // TODO: Improve this test. Somehow force it to go to fat locked. But that needs another thread. - + // Test done. #else LOG(INFO) << "Skipping unlock_object as I don't know how to do that on " << kRuntimeISA; // Force-print to std::cout so it's also outside the logcat. @@ -548,6 +586,9 @@ TEST_F(StubTest, UnlockObject) { #endif } +TEST_F(StubTest, UnlockObject) { + TestUnlockObject(this); +} #if defined(__i386__) || defined(__arm__) || defined(__aarch64__) || defined(__x86_64__) extern "C" void art_quick_check_cast(void); diff --git a/runtime/arch/x86_64/quick_entrypoints_x86_64.S b/runtime/arch/x86_64/quick_entrypoints_x86_64.S index a55dbb6b2..b886fb0f9 100644 --- a/runtime/arch/x86_64/quick_entrypoints_x86_64.S +++ b/runtime/arch/x86_64/quick_entrypoints_x86_64.S @@ -697,8 +697,8 @@ DEFINE_FUNCTION art_quick_unlock_object jz .Lslow_unlock movl LOCK_WORD_OFFSET(%edi), %ecx // ecx := lock word movl %gs:THREAD_ID_OFFSET, %edx // edx := thread id - test %ecx, %ecx - jb .Lslow_unlock // lock word contains a monitor + test LITERAL(0xC0000000), %ecx + jnz .Lslow_unlock // lock word contains a monitor cmpw %cx, %dx // does the thread id match? jne .Lslow_unlock cmpl LITERAL(65536), %ecx |