summaryrefslogtreecommitdiffstats
path: root/compiler/optimizing/code_generator_arm64.cc
diff options
context:
space:
mode:
Diffstat (limited to 'compiler/optimizing/code_generator_arm64.cc')
-rw-r--r--compiler/optimizing/code_generator_arm64.cc360
1 files changed, 308 insertions, 52 deletions
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index c7517d3abc..271eb82ee6 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -17,10 +17,12 @@
#include "code_generator_arm64.h"
#include "entrypoints/quick/quick_entrypoints.h"
+#include "entrypoints/quick/quick_entrypoints_enum.h"
#include "gc/accounting/card_table.h"
#include "mirror/array-inl.h"
#include "mirror/art_method.h"
#include "mirror/class.h"
+#include "offsets.h"
#include "thread.h"
#include "utils/arm64/assembler_arm64.h"
#include "utils/assembler.h"
@@ -38,7 +40,9 @@ namespace art {
namespace arm64 {
-static constexpr bool kExplicitStackOverflowCheck = false;
+// TODO: Tune the use of Load-Acquire, Store-Release vs Data Memory Barriers.
+// For now we prefer the use of load-acquire, store-release over explicit memory barriers.
+static constexpr bool kUseAcquireRelease = true;
static constexpr size_t kHeapRefSize = sizeof(mirror::HeapReference<mirror::Object>);
static constexpr int kCurrentMethodStackOffset = 0;
@@ -233,8 +237,9 @@ Location ARM64ReturnLocation(Primitive::Type return_type) {
static const Register kRuntimeParameterCoreRegisters[] = { x0, x1, x2, x3, x4, x5, x6, x7 };
static constexpr size_t kRuntimeParameterCoreRegistersLength =
arraysize(kRuntimeParameterCoreRegisters);
-static const FPRegister kRuntimeParameterFpuRegisters[] = { };
-static constexpr size_t kRuntimeParameterFpuRegistersLength = 0;
+static const FPRegister kRuntimeParameterFpuRegisters[] = { d0, d1, d2, d3, d4, d5, d6, d7 };
+static constexpr size_t kRuntimeParameterFpuRegistersLength =
+ arraysize(kRuntimeParameterCoreRegisters);
class InvokeRuntimeCallingConvention : public CallingConvention<Register, FPRegister> {
public:
@@ -294,6 +299,7 @@ class BoundsCheckSlowPathARM64 : public SlowPathCodeARM64 {
length_location_, LocationFrom(calling_convention.GetRegisterAt(1)));
arm64_codegen->InvokeRuntime(
QUICK_ENTRY_POINT(pThrowArrayBounds), instruction_, instruction_->GetDexPc());
+ CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>();
}
private:
@@ -313,6 +319,7 @@ class DivZeroCheckSlowPathARM64 : public SlowPathCodeARM64 {
__ Bind(GetEntryLabel());
arm64_codegen->InvokeRuntime(
QUICK_ENTRY_POINT(pThrowDivZero), instruction_, instruction_->GetDexPc());
+ CheckEntrypointTypes<kQuickThrowDivZero, void, void>();
}
private:
@@ -343,6 +350,11 @@ class LoadClassSlowPathARM64 : public SlowPathCodeARM64 {
int32_t entry_point_offset = do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage)
: QUICK_ENTRY_POINT(pInitializeType);
arm64_codegen->InvokeRuntime(entry_point_offset, at_, dex_pc_);
+ if (do_clinit_) {
+ CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t, mirror::ArtMethod*>();
+ } else {
+ CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t, mirror::ArtMethod*>();
+ }
// Move the class to the desired location.
Location out = locations->Out();
@@ -386,10 +398,11 @@ class LoadStringSlowPathARM64 : public SlowPathCodeARM64 {
codegen->SaveLiveRegisters(locations);
InvokeRuntimeCallingConvention calling_convention;
- arm64_codegen->LoadCurrentMethod(calling_convention.GetRegisterAt(0).W());
- __ Mov(calling_convention.GetRegisterAt(1).W(), instruction_->GetStringIndex());
+ arm64_codegen->LoadCurrentMethod(calling_convention.GetRegisterAt(1).W());
+ __ Mov(calling_convention.GetRegisterAt(0).W(), instruction_->GetStringIndex());
arm64_codegen->InvokeRuntime(
QUICK_ENTRY_POINT(pResolveString), instruction_, instruction_->GetDexPc());
+ CheckEntrypointTypes<kQuickResolveString, void*, uint32_t, mirror::ArtMethod*>();
Primitive::Type type = instruction_->GetType();
arm64_codegen->MoveLocation(locations->Out(), calling_convention.GetReturnLocation(type), type);
@@ -412,6 +425,7 @@ class NullCheckSlowPathARM64 : public SlowPathCodeARM64 {
__ Bind(GetEntryLabel());
arm64_codegen->InvokeRuntime(
QUICK_ENTRY_POINT(pThrowNullPointer), instruction_, instruction_->GetDexPc());
+ CheckEntrypointTypes<kQuickThrowNullPointer, void, void>();
}
private:
@@ -428,6 +442,7 @@ class StackOverflowCheckSlowPathARM64 : public SlowPathCodeARM64 {
CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
__ Bind(GetEntryLabel());
arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowStackOverflow), nullptr, 0);
+ CheckEntrypointTypes<kQuickThrowStackOverflow, void, void*>();
}
private:
@@ -446,6 +461,7 @@ class SuspendCheckSlowPathARM64 : public SlowPathCodeARM64 {
codegen->SaveLiveRegisters(instruction_->GetLocations());
arm64_codegen->InvokeRuntime(
QUICK_ENTRY_POINT(pTestSuspend), instruction_, instruction_->GetDexPc());
+ CheckEntrypointTypes<kQuickTestSuspend, void, void>();
codegen->RestoreLiveRegisters(instruction_->GetLocations());
if (successor_ == nullptr) {
__ B(GetReturnLabel());
@@ -502,9 +518,12 @@ class TypeCheckSlowPathARM64 : public SlowPathCodeARM64 {
Primitive::Type ret_type = instruction_->GetType();
Location ret_loc = calling_convention.GetReturnLocation(ret_type);
arm64_codegen->MoveLocation(locations->Out(), ret_loc, ret_type);
+ CheckEntrypointTypes<kQuickInstanceofNonTrivial, uint32_t,
+ const mirror::Class*, const mirror::Class*>();
} else {
DCHECK(instruction_->IsCheckCast());
arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast), instruction_, dex_pc_);
+ CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>();
}
codegen->RestoreLiveRegisters(locations);
@@ -543,11 +562,12 @@ Location InvokeDexCallingConventionVisitor::GetNextLocation(Primitive::Type type
return next_location;
}
-CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph)
+CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph, const CompilerOptions& compiler_options)
: CodeGenerator(graph,
kNumberOfAllocatableRegisters,
kNumberOfAllocatableFPRegisters,
- kNumberOfAllocatableRegisterPairs),
+ kNumberOfAllocatableRegisterPairs,
+ compiler_options),
block_labels_(nullptr),
location_builder_(graph, this),
instruction_visitor_(graph, this),
@@ -585,17 +605,17 @@ void CodeGeneratorARM64::GenerateFrameEntry() {
if (do_overflow_check) {
UseScratchRegisterScope temps(GetVIXLAssembler());
Register temp = temps.AcquireX();
- if (kExplicitStackOverflowCheck) {
+ if (GetCompilerOptions().GetImplicitStackOverflowChecks()) {
+ __ Add(temp, sp, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm64)));
+ __ Ldr(wzr, MemOperand(temp, 0));
+ RecordPcInfo(nullptr, 0);
+ } else {
SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) StackOverflowCheckSlowPathARM64();
AddSlowPath(slow_path);
__ Ldr(temp, MemOperand(tr, Thread::StackEndOffset<kArm64WordSize>().Int32Value()));
__ Cmp(sp, temp);
__ B(lo, slow_path->GetEntryLabel());
- } else {
- __ Add(temp, sp, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm64)));
- __ Ldr(wzr, MemOperand(temp, 0));
- RecordPcInfo(nullptr, 0);
}
}
@@ -949,8 +969,8 @@ void CodeGeneratorARM64::SwapLocations(Location loc1, Location loc2) {
}
void CodeGeneratorARM64::Load(Primitive::Type type,
- vixl::CPURegister dst,
- const vixl::MemOperand& src) {
+ CPURegister dst,
+ const MemOperand& src) {
switch (type) {
case Primitive::kPrimBoolean:
__ Ldrb(Register(dst), src);
@@ -969,7 +989,7 @@ void CodeGeneratorARM64::Load(Primitive::Type type,
case Primitive::kPrimLong:
case Primitive::kPrimFloat:
case Primitive::kPrimDouble:
- DCHECK(dst.Is64Bits() == Is64BitType(type));
+ DCHECK_EQ(dst.Is64Bits(), Is64BitType(type));
__ Ldr(dst, src);
break;
case Primitive::kPrimVoid:
@@ -977,31 +997,130 @@ void CodeGeneratorARM64::Load(Primitive::Type type,
}
}
+void CodeGeneratorARM64::LoadAcquire(HInstruction* instruction,
+ CPURegister dst,
+ const MemOperand& src) {
+ UseScratchRegisterScope temps(GetVIXLAssembler());
+ Register temp_base = temps.AcquireX();
+ Primitive::Type type = instruction->GetType();
+
+ DCHECK(!src.IsRegisterOffset());
+ DCHECK(!src.IsPreIndex());
+ DCHECK(!src.IsPostIndex());
+
+ // TODO(vixl): Let the MacroAssembler handle MemOperand.
+ __ Add(temp_base, src.base(), src.offset());
+ MemOperand base = MemOperand(temp_base);
+ switch (type) {
+ case Primitive::kPrimBoolean:
+ __ Ldarb(Register(dst), base);
+ MaybeRecordImplicitNullCheck(instruction);
+ break;
+ case Primitive::kPrimByte:
+ __ Ldarb(Register(dst), base);
+ MaybeRecordImplicitNullCheck(instruction);
+ __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte);
+ break;
+ case Primitive::kPrimChar:
+ __ Ldarh(Register(dst), base);
+ MaybeRecordImplicitNullCheck(instruction);
+ break;
+ case Primitive::kPrimShort:
+ __ Ldarh(Register(dst), base);
+ MaybeRecordImplicitNullCheck(instruction);
+ __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte);
+ break;
+ case Primitive::kPrimInt:
+ case Primitive::kPrimNot:
+ case Primitive::kPrimLong:
+ DCHECK_EQ(dst.Is64Bits(), Is64BitType(type));
+ __ Ldar(Register(dst), base);
+ MaybeRecordImplicitNullCheck(instruction);
+ break;
+ case Primitive::kPrimFloat:
+ case Primitive::kPrimDouble: {
+ DCHECK(dst.IsFPRegister());
+ DCHECK_EQ(dst.Is64Bits(), Is64BitType(type));
+
+ Register temp = dst.Is64Bits() ? temps.AcquireX() : temps.AcquireW();
+ __ Ldar(temp, base);
+ MaybeRecordImplicitNullCheck(instruction);
+ __ Fmov(FPRegister(dst), temp);
+ break;
+ }
+ case Primitive::kPrimVoid:
+ LOG(FATAL) << "Unreachable type " << type;
+ }
+}
+
void CodeGeneratorARM64::Store(Primitive::Type type,
- vixl::CPURegister rt,
- const vixl::MemOperand& dst) {
+ CPURegister src,
+ const MemOperand& dst) {
switch (type) {
case Primitive::kPrimBoolean:
case Primitive::kPrimByte:
- __ Strb(Register(rt), dst);
+ __ Strb(Register(src), dst);
break;
case Primitive::kPrimChar:
case Primitive::kPrimShort:
- __ Strh(Register(rt), dst);
+ __ Strh(Register(src), dst);
break;
case Primitive::kPrimInt:
case Primitive::kPrimNot:
case Primitive::kPrimLong:
case Primitive::kPrimFloat:
case Primitive::kPrimDouble:
- DCHECK(rt.Is64Bits() == Is64BitType(type));
- __ Str(rt, dst);
+ DCHECK_EQ(src.Is64Bits(), Is64BitType(type));
+ __ Str(src, dst);
break;
case Primitive::kPrimVoid:
LOG(FATAL) << "Unreachable type " << type;
}
}
+void CodeGeneratorARM64::StoreRelease(Primitive::Type type,
+ CPURegister src,
+ const MemOperand& dst) {
+ UseScratchRegisterScope temps(GetVIXLAssembler());
+ Register temp_base = temps.AcquireX();
+
+ DCHECK(!dst.IsRegisterOffset());
+ DCHECK(!dst.IsPreIndex());
+ DCHECK(!dst.IsPostIndex());
+
+ // TODO(vixl): Let the MacroAssembler handle this.
+ __ Add(temp_base, dst.base(), dst.offset());
+ MemOperand base = MemOperand(temp_base);
+ switch (type) {
+ case Primitive::kPrimBoolean:
+ case Primitive::kPrimByte:
+ __ Stlrb(Register(src), base);
+ break;
+ case Primitive::kPrimChar:
+ case Primitive::kPrimShort:
+ __ Stlrh(Register(src), base);
+ break;
+ case Primitive::kPrimInt:
+ case Primitive::kPrimNot:
+ case Primitive::kPrimLong:
+ DCHECK_EQ(src.Is64Bits(), Is64BitType(type));
+ __ Stlr(Register(src), base);
+ break;
+ case Primitive::kPrimFloat:
+ case Primitive::kPrimDouble: {
+ DCHECK(src.IsFPRegister());
+ DCHECK_EQ(src.Is64Bits(), Is64BitType(type));
+
+ Register temp = src.Is64Bits() ? temps.AcquireX() : temps.AcquireW();
+ __ Fmov(temp, FPRegister(src));
+ __ Stlr(temp, base);
+ break;
+ }
+ case Primitive::kPrimVoid:
+ LOG(FATAL) << "Unreachable type " << type;
+ }
+}
+
void CodeGeneratorARM64::LoadCurrentMethod(vixl::Register current_method) {
DCHECK(current_method.IsW());
__ Ldr(current_method, MemOperand(sp, kCurrentMethodStackOffset));
@@ -1026,14 +1145,47 @@ void InstructionCodeGeneratorARM64::GenerateClassInitializationCheck(SlowPathCod
vixl::Register class_reg) {
UseScratchRegisterScope temps(GetVIXLAssembler());
Register temp = temps.AcquireW();
- __ Ldr(temp, HeapOperand(class_reg, mirror::Class::StatusOffset()));
- __ Cmp(temp, mirror::Class::kStatusInitialized);
- __ B(lt, slow_path->GetEntryLabel());
+ size_t status_offset = mirror::Class::StatusOffset().SizeValue();
+
// Even if the initialized flag is set, we need to ensure consistent memory ordering.
- __ Dmb(InnerShareable, BarrierReads);
+ if (kUseAcquireRelease) {
+ // TODO(vixl): Let the MacroAssembler handle MemOperand.
+ __ Add(temp, class_reg, status_offset);
+ __ Ldar(temp, HeapOperand(temp));
+ __ Cmp(temp, mirror::Class::kStatusInitialized);
+ __ B(lt, slow_path->GetEntryLabel());
+ } else {
+ __ Ldr(temp, HeapOperand(class_reg, status_offset));
+ __ Cmp(temp, mirror::Class::kStatusInitialized);
+ __ B(lt, slow_path->GetEntryLabel());
+ __ Dmb(InnerShareable, BarrierReads);
+ }
__ Bind(slow_path->GetExitLabel());
}
+void InstructionCodeGeneratorARM64::GenerateMemoryBarrier(MemBarrierKind kind) {
+ BarrierType type = BarrierAll;
+
+ switch (kind) {
+ case MemBarrierKind::kAnyAny:
+ case MemBarrierKind::kAnyStore: {
+ type = BarrierAll;
+ break;
+ }
+ case MemBarrierKind::kLoadAny: {
+ type = BarrierReads;
+ break;
+ }
+ case MemBarrierKind::kStoreStore: {
+ type = BarrierWrites;
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected memory barrier " << kind;
+ }
+ __ Dmb(InnerShareable, type);
+}
+
void InstructionCodeGeneratorARM64::GenerateSuspendCheck(HSuspendCheck* instruction,
HBasicBlock* successor) {
SuspendCheckSlowPathARM64* slow_path =
@@ -1254,6 +1406,7 @@ void InstructionCodeGeneratorARM64::VisitArrayGet(HArrayGet* instruction) {
}
codegen_->Load(type, OutputCPURegister(instruction), source);
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
}
void LocationsBuilderARM64::VisitArrayLength(HArrayLength* instruction) {
@@ -1265,6 +1418,7 @@ void LocationsBuilderARM64::VisitArrayLength(HArrayLength* instruction) {
void InstructionCodeGeneratorARM64::VisitArrayLength(HArrayLength* instruction) {
__ Ldr(OutputRegister(instruction),
HeapOperand(InputRegisterAt(instruction, 0), mirror::Array::LengthOffset()));
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
}
void LocationsBuilderARM64::VisitArraySet(HArraySet* instruction) {
@@ -1288,7 +1442,7 @@ void InstructionCodeGeneratorARM64::VisitArraySet(HArraySet* instruction) {
Primitive::Type value_type = instruction->GetComponentType();
if (value_type == Primitive::kPrimNot) {
codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject), instruction, instruction->GetDexPc());
-
+ CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>();
} else {
LocationSummary* locations = instruction->GetLocations();
Register obj = InputRegisterAt(instruction, 0);
@@ -1309,6 +1463,7 @@ void InstructionCodeGeneratorARM64::VisitArraySet(HArraySet* instruction) {
}
codegen_->Store(value_type, value, destination);
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
}
}
@@ -1660,28 +1815,60 @@ void InstructionCodeGeneratorARM64::VisitIf(HIf* if_instr) {
}
void LocationsBuilderARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
- LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
+ LocationSummary* locations =
+ new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
locations->SetInAt(0, Location::RequiresRegister());
locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
}
void InstructionCodeGeneratorARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
MemOperand field = HeapOperand(InputRegisterAt(instruction, 0), instruction->GetFieldOffset());
- codegen_->Load(instruction->GetType(), OutputCPURegister(instruction), field);
+
+ if (instruction->IsVolatile()) {
+ if (kUseAcquireRelease) {
+ // NB: LoadAcquire will record the pc info if needed.
+ codegen_->LoadAcquire(instruction, OutputCPURegister(instruction), field);
+ } else {
+ codegen_->Load(instruction->GetType(), OutputCPURegister(instruction), field);
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
+ // For IRIW sequential consistency kLoadAny is not sufficient.
+ GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
+ }
+ } else {
+ codegen_->Load(instruction->GetType(), OutputCPURegister(instruction), field);
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
+ }
}
void LocationsBuilderARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
- LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
+ LocationSummary* locations =
+ new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
locations->SetInAt(0, Location::RequiresRegister());
locations->SetInAt(1, Location::RequiresRegister());
}
void InstructionCodeGeneratorARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
- Primitive::Type field_type = instruction->GetFieldType();
- CPURegister value = InputCPURegisterAt(instruction, 1);
Register obj = InputRegisterAt(instruction, 0);
- codegen_->Store(field_type, value, HeapOperand(obj, instruction->GetFieldOffset()));
- if (field_type == Primitive::kPrimNot) {
+ CPURegister value = InputCPURegisterAt(instruction, 1);
+ Offset offset = instruction->GetFieldOffset();
+ Primitive::Type field_type = instruction->GetFieldType();
+
+ if (instruction->IsVolatile()) {
+ if (kUseAcquireRelease) {
+ codegen_->StoreRelease(field_type, value, HeapOperand(obj, offset));
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
+ } else {
+ GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
+ codegen_->Store(field_type, value, HeapOperand(obj, offset));
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
+ GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
+ }
+ } else {
+ codegen_->Store(field_type, value, HeapOperand(obj, offset));
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
+ }
+
+ if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
codegen_->MarkGCCard(obj, Register(value));
}
}
@@ -1692,7 +1879,8 @@ void LocationsBuilderARM64::VisitInstanceOf(HInstanceOf* instruction) {
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
locations->SetInAt(0, Location::RequiresRegister());
locations->SetInAt(1, Location::RequiresRegister());
- locations->SetOut(Location::RequiresRegister(), true); // The output does overlap inputs.
+ // The output does overlap inputs.
+ locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
}
void InstructionCodeGeneratorARM64::VisitInstanceOf(HInstanceOf* instruction) {
@@ -1782,6 +1970,7 @@ void InstructionCodeGeneratorARM64::VisitInvokeInterface(HInvokeInterface* invok
} else {
__ Ldr(temp, HeapOperandFrom(receiver, class_offset));
}
+ codegen_->MaybeRecordImplicitNullCheck(invoke);
// temp = temp->GetImtEntryAt(method_offset);
__ Ldr(temp, HeapOperand(temp, method_offset));
// lr = temp->GetEntryPoint();
@@ -1805,7 +1994,7 @@ void InstructionCodeGeneratorARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDir
// Make sure that ArtMethod* is passed in W0 as per the calling convention
DCHECK(temp.Is(w0));
size_t index_in_cache = mirror::Array::DataOffset(kHeapRefSize).SizeValue() +
- invoke->GetIndexInDexCache() * kHeapRefSize;
+ invoke->GetDexMethodIndex() * kHeapRefSize;
// TODO: Implement all kinds of calls:
// 1) boot -> boot
@@ -1847,6 +2036,7 @@ void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
DCHECK(receiver.IsRegister());
__ Ldr(temp, HeapOperandFrom(receiver, class_offset));
}
+ codegen_->MaybeRecordImplicitNullCheck(invoke);
// temp = temp->GetMethodAt(method_offset);
__ Ldr(temp, HeapOperand(temp, method_offset));
// lr = temp->GetEntryPoint();
@@ -1959,6 +2149,7 @@ void InstructionCodeGeneratorARM64::VisitMonitorOperation(HMonitorOperation* ins
? QUICK_ENTRY_POINT(pLockObject) : QUICK_ENTRY_POINT(pUnlockObject),
instruction,
instruction->GetDexPc());
+ CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>();
}
void LocationsBuilderARM64::VisitMul(HMul* mul) {
@@ -2044,9 +2235,11 @@ void LocationsBuilderARM64::VisitNewArray(HNewArray* instruction) {
new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
InvokeRuntimeCallingConvention calling_convention;
locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(0)));
- locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(1)));
+ locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(2)));
locations->SetOut(LocationFrom(x0));
- locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(2)));
+ locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1)));
+ CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck,
+ void*, uint32_t, int32_t, mirror::ArtMethod*>();
}
void InstructionCodeGeneratorARM64::VisitNewArray(HNewArray* instruction) {
@@ -2055,11 +2248,13 @@ void InstructionCodeGeneratorARM64::VisitNewArray(HNewArray* instruction) {
Register type_index = RegisterFrom(locations->GetTemp(0), Primitive::kPrimInt);
DCHECK(type_index.Is(w0));
Register current_method = RegisterFrom(locations->GetTemp(1), Primitive::kPrimNot);
- DCHECK(current_method.Is(w1));
+ DCHECK(current_method.Is(w2));
codegen_->LoadCurrentMethod(current_method);
__ Mov(type_index, instruction->GetTypeIndex());
codegen_->InvokeRuntime(
QUICK_ENTRY_POINT(pAllocArrayWithAccessCheck), instruction, instruction->GetDexPc());
+ CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck,
+ void*, uint32_t, int32_t, mirror::ArtMethod*>();
}
void LocationsBuilderARM64::VisitNewInstance(HNewInstance* instruction) {
@@ -2069,6 +2264,7 @@ void LocationsBuilderARM64::VisitNewInstance(HNewInstance* instruction) {
locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(0)));
locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(1)));
locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
+ CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, mirror::ArtMethod*>();
}
void InstructionCodeGeneratorARM64::VisitNewInstance(HNewInstance* instruction) {
@@ -2081,6 +2277,7 @@ void InstructionCodeGeneratorARM64::VisitNewInstance(HNewInstance* instruction)
__ Mov(type_index, instruction->GetTypeIndex());
codegen_->InvokeRuntime(
QUICK_ENTRY_POINT(pAllocObjectWithAccessCheck), instruction, instruction->GetDexPc());
+ CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, mirror::ArtMethod*>();
}
void LocationsBuilderARM64::VisitNot(HNot* instruction) {
@@ -2114,18 +2311,31 @@ void LocationsBuilderARM64::VisitNullCheck(HNullCheck* instruction) {
}
}
-void InstructionCodeGeneratorARM64::VisitNullCheck(HNullCheck* instruction) {
+void InstructionCodeGeneratorARM64::GenerateImplicitNullCheck(HNullCheck* instruction) {
+ if (codegen_->CanMoveNullCheckToUser(instruction)) {
+ return;
+ }
+ Location obj = instruction->GetLocations()->InAt(0);
+
+ __ Ldr(wzr, HeapOperandFrom(obj, Offset(0)));
+ codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
+}
+
+void InstructionCodeGeneratorARM64::GenerateExplicitNullCheck(HNullCheck* instruction) {
SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM64(instruction);
codegen_->AddSlowPath(slow_path);
LocationSummary* locations = instruction->GetLocations();
Location obj = locations->InAt(0);
- if (obj.IsRegister()) {
- __ Cbz(RegisterFrom(obj, instruction->InputAt(0)->GetType()), slow_path->GetEntryLabel());
+
+ __ Cbz(RegisterFrom(obj, instruction->InputAt(0)->GetType()), slow_path->GetEntryLabel());
+}
+
+void InstructionCodeGeneratorARM64::VisitNullCheck(HNullCheck* instruction) {
+ if (codegen_->GetCompilerOptions().GetImplicitNullChecks()) {
+ GenerateImplicitNullCheck(instruction);
} else {
- DCHECK(obj.IsConstant()) << obj;
- DCHECK_EQ(obj.GetConstant()->AsIntConstant()->GetValue(), 0);
- __ B(slow_path->GetEntryLabel());
+ GenerateExplicitNullCheck(instruction);
}
}
@@ -2175,9 +2385,12 @@ void InstructionCodeGeneratorARM64::VisitPhi(HPhi* instruction) {
}
void LocationsBuilderARM64::VisitRem(HRem* rem) {
- LocationSummary* locations =
- new (GetGraph()->GetArena()) LocationSummary(rem, LocationSummary::kNoCall);
- switch (rem->GetResultType()) {
+ Primitive::Type type = rem->GetResultType();
+ LocationSummary::CallKind call_kind = IsFPType(type) ? LocationSummary::kCall
+ : LocationSummary::kNoCall;
+ LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
+
+ switch (type) {
case Primitive::kPrimInt:
case Primitive::kPrimLong:
locations->SetInAt(0, Location::RequiresRegister());
@@ -2185,13 +2398,24 @@ void LocationsBuilderARM64::VisitRem(HRem* rem) {
locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
break;
+ case Primitive::kPrimFloat:
+ case Primitive::kPrimDouble: {
+ InvokeRuntimeCallingConvention calling_convention;
+ locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0)));
+ locations->SetInAt(1, LocationFrom(calling_convention.GetFpuRegisterAt(1)));
+ locations->SetOut(calling_convention.GetReturnLocation(type));
+
+ break;
+ }
+
default:
- LOG(FATAL) << "Unexpected rem type " << rem->GetResultType();
+ LOG(FATAL) << "Unexpected rem type " << type;
}
}
void InstructionCodeGeneratorARM64::VisitRem(HRem* rem) {
Primitive::Type type = rem->GetResultType();
+
switch (type) {
case Primitive::kPrimInt:
case Primitive::kPrimLong: {
@@ -2206,6 +2430,14 @@ void InstructionCodeGeneratorARM64::VisitRem(HRem* rem) {
break;
}
+ case Primitive::kPrimFloat:
+ case Primitive::kPrimDouble: {
+ int32_t entry_offset = (type == Primitive::kPrimFloat) ? QUICK_ENTRY_POINT(pFmodf)
+ : QUICK_ENTRY_POINT(pFmod);
+ codegen_->InvokeRuntime(entry_offset, rem, rem->GetDexPc());
+ break;
+ }
+
default:
LOG(FATAL) << "Unexpected rem type " << type;
}
@@ -2294,7 +2526,19 @@ void LocationsBuilderARM64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
void InstructionCodeGeneratorARM64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
MemOperand field = HeapOperand(InputRegisterAt(instruction, 0), instruction->GetFieldOffset());
- codegen_->Load(instruction->GetType(), OutputCPURegister(instruction), field);
+
+ if (instruction->IsVolatile()) {
+ if (kUseAcquireRelease) {
+ // NB: LoadAcquire will record the pc info if needed.
+ codegen_->LoadAcquire(instruction, OutputCPURegister(instruction), field);
+ } else {
+ codegen_->Load(instruction->GetType(), OutputCPURegister(instruction), field);
+ // For IRIW sequential consistency kLoadAny is not sufficient.
+ GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
+ }
+ } else {
+ codegen_->Load(instruction->GetType(), OutputCPURegister(instruction), field);
+ }
}
void LocationsBuilderARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
@@ -2305,13 +2549,24 @@ void LocationsBuilderARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
}
void InstructionCodeGeneratorARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
- CPURegister value = InputCPURegisterAt(instruction, 1);
Register cls = InputRegisterAt(instruction, 0);
+ CPURegister value = InputCPURegisterAt(instruction, 1);
Offset offset = instruction->GetFieldOffset();
Primitive::Type field_type = instruction->GetFieldType();
- codegen_->Store(field_type, value, HeapOperand(cls, offset));
- if (field_type == Primitive::kPrimNot) {
+ if (instruction->IsVolatile()) {
+ if (kUseAcquireRelease) {
+ codegen_->StoreRelease(field_type, value, HeapOperand(cls, offset));
+ } else {
+ GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
+ codegen_->Store(field_type, value, HeapOperand(cls, offset));
+ GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
+ }
+ } else {
+ codegen_->Store(field_type, value, HeapOperand(cls, offset));
+ }
+
+ if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
codegen_->MarkGCCard(cls, Register(value));
}
}
@@ -2353,6 +2608,7 @@ void LocationsBuilderARM64::VisitThrow(HThrow* instruction) {
void InstructionCodeGeneratorARM64::VisitThrow(HThrow* instruction) {
codegen_->InvokeRuntime(
QUICK_ENTRY_POINT(pDeliverException), instruction, instruction->GetDexPc());
+ CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>();
}
void LocationsBuilderARM64::VisitTypeConversion(HTypeConversion* conversion) {
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-25 19:17:14 +0000