diff options
| -rw-r--r-- | compiler/optimizing/code_generator_arm.cc | 18 | ||||
| -rw-r--r-- | compiler/optimizing/code_generator_arm64.cc | 813 | ||||
| -rw-r--r-- | compiler/optimizing/code_generator_arm64.h | 36 | ||||
| -rw-r--r-- | test/Android.run-test.mk | 130 |
4 files changed, 685 insertions, 312 deletions
diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index 1debaa555e..39543a24a0 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -80,7 +80,7 @@ class NullCheckSlowPathARM : public SlowPathCodeARM { public: explicit NullCheckSlowPathARM(HNullCheck* instruction) : instruction_(instruction) {} - virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); __ Bind(GetEntryLabel()); arm_codegen->InvokeRuntime( @@ -96,7 +96,7 @@ class DivZeroCheckSlowPathARM : public SlowPathCodeARM { public: explicit DivZeroCheckSlowPathARM(HDivZeroCheck* instruction) : instruction_(instruction) {} - virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); __ Bind(GetEntryLabel()); arm_codegen->InvokeRuntime( @@ -112,7 +112,7 @@ class StackOverflowCheckSlowPathARM : public SlowPathCodeARM { public: StackOverflowCheckSlowPathARM() {} - virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { __ Bind(GetEntryLabel()); __ LoadFromOffset(kLoadWord, PC, TR, QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pThrowStackOverflow).Int32Value()); @@ -124,10 +124,10 @@ class StackOverflowCheckSlowPathARM : public SlowPathCodeARM { class SuspendCheckSlowPathARM : public SlowPathCodeARM { public: - explicit SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor) + SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor) : instruction_(instruction), successor_(successor) {} - virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); __ Bind(GetEntryLabel()); codegen->SaveLiveRegisters(instruction_->GetLocations()); @@ -166,7 +166,7 @@ class BoundsCheckSlowPathARM : public SlowPathCodeARM { index_location_(index_location), length_location_(length_location) {} - virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); __ Bind(GetEntryLabel()); // We're moving two locations to locations that could overlap, so we need a parallel @@ -199,7 +199,7 @@ class LoadClassSlowPathARM : public SlowPathCodeARM { DCHECK(at->IsLoadClass() || at->IsClinitCheck()); } - virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = at_->GetLocations(); CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); @@ -245,7 +245,7 @@ class LoadStringSlowPathARM : public SlowPathCodeARM { public: explicit LoadStringSlowPathARM(HLoadString* instruction) : instruction_(instruction) {} - virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); @@ -281,7 +281,7 @@ class TypeCheckSlowPathARM : public SlowPathCodeARM { object_class_(object_class), dex_pc_(dex_pc) {} - virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); DCHECK(instruction_->IsCheckCast() || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index 5432882968..fc9bdba52b 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -48,18 +48,28 @@ bool IsFPType(Primitive::Type type) { return type == Primitive::kPrimFloat || type == Primitive::kPrimDouble; } +bool IsIntegralType(Primitive::Type type) { + switch (type) { + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + return true; + default: + return false; + } +} + bool Is64BitType(Primitive::Type type) { return type == Primitive::kPrimLong || type == Primitive::kPrimDouble; } // Convenience helpers to ease conversion to and from VIXL operands. +static_assert((SP == 31) && (WSP == 31) && (XZR == 32) && (WZR == 32), + "Unexpected values for register codes."); int VIXLRegCodeFromART(int code) { - // TODO: static check? - DCHECK_EQ(SP, 31); - DCHECK_EQ(WSP, 31); - DCHECK_EQ(XZR, 32); - DCHECK_EQ(WZR, 32); if (code == SP) { return vixl::kSPRegInternalCode; } @@ -70,11 +80,6 @@ int VIXLRegCodeFromART(int code) { } int ARTRegCodeFromVIXL(int code) { - // TODO: static check? - DCHECK_EQ(SP, 31); - DCHECK_EQ(WSP, 31); - DCHECK_EQ(XZR, 32); - DCHECK_EQ(WZR, 32); if (code == vixl::kSPRegInternalCode) { return SP; } @@ -128,6 +133,17 @@ FPRegister InputFPRegisterAt(HInstruction* instr, int input_index) { instr->InputAt(input_index)->GetType()); } +CPURegister OutputCPURegister(HInstruction* instr) { + return IsFPType(instr->GetType()) ? static_cast<CPURegister>(OutputFPRegister(instr)) + : static_cast<CPURegister>(OutputRegister(instr)); +} + +CPURegister InputCPURegisterAt(HInstruction* instr, int index) { + return IsFPType(instr->InputAt(index)->GetType()) + ? static_cast<CPURegister>(InputFPRegisterAt(instr, index)) + : static_cast<CPURegister>(InputRegisterAt(instr, index)); +} + int64_t Int64ConstantFrom(Location location) { HConstant* instr = location.GetConstant(); return instr->IsIntConstant() ? instr->AsIntConstant()->GetValue() @@ -151,14 +167,18 @@ MemOperand StackOperandFrom(Location location) { return MemOperand(sp, location.GetStackIndex()); } -MemOperand HeapOperand(const Register& base, Offset offset) { +MemOperand HeapOperand(const Register& base, size_t offset) { // A heap reference must be 32bit, so fit in a W register. DCHECK(base.IsW()); - return MemOperand(base.X(), offset.SizeValue()); + return MemOperand(base.X(), offset); } -MemOperand HeapOperandFrom(Location location, Primitive::Type type, Offset offset) { - return HeapOperand(RegisterFrom(location, type), offset); +MemOperand HeapOperand(const Register& base, Offset offset) { + return HeapOperand(base, offset.SizeValue()); +} + +MemOperand HeapOperandFrom(Location location, Offset offset) { + return HeapOperand(RegisterFrom(location, Primitive::kPrimNot), offset); } Location LocationFrom(const Register& reg) { @@ -227,7 +247,8 @@ Location InvokeRuntimeCallingConvention::GetReturnLocation(Primitive::Type retur return ARM64ReturnLocation(return_type); } -#define __ reinterpret_cast<Arm64Assembler*>(codegen->GetAssembler())->vixl_masm_-> +#define __ down_cast<CodeGeneratorARM64*>(codegen)->GetVIXLAssembler()-> +#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, x).Int32Value() class SlowPathCodeARM64 : public SlowPathCode { public: @@ -245,45 +266,125 @@ class SlowPathCodeARM64 : public SlowPathCode { class BoundsCheckSlowPathARM64 : public SlowPathCodeARM64 { public: - explicit BoundsCheckSlowPathARM64(HBoundsCheck* instruction, - Location index_location, - Location length_location) - : instruction_(instruction), - index_location_(index_location), - length_location_(length_location) {} - - virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorARM64* arm64_codegen = reinterpret_cast<CodeGeneratorARM64*>(codegen); + BoundsCheckSlowPathARM64() {} + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { __ Bind(GetEntryLabel()); + __ Brk(__LINE__); // TODO: Unimplemented BoundsCheckSlowPathARM64. + } + + private: + DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM64); +}; + +class DivZeroCheckSlowPathARM64 : public SlowPathCodeARM64 { + public: + explicit DivZeroCheckSlowPathARM64(HDivZeroCheck* instruction) : instruction_(instruction) {} + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); + __ Bind(GetEntryLabel()); + arm64_codegen->InvokeRuntime( + QUICK_ENTRY_POINT(pThrowDivZero), instruction_, instruction_->GetDexPc()); + } + + private: + HDivZeroCheck* const instruction_; + DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM64); +}; + +class LoadClassSlowPathARM64 : public SlowPathCodeARM64 { + public: + LoadClassSlowPathARM64(HLoadClass* cls, + HInstruction* at, + uint32_t dex_pc, + bool do_clinit) + : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) { + DCHECK(at->IsLoadClass() || at->IsClinitCheck()); + } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = at_->GetLocations(); + CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); + + __ Bind(GetEntryLabel()); + codegen->SaveLiveRegisters(locations); + InvokeRuntimeCallingConvention calling_convention; - arm64_codegen->MoveHelper(LocationFrom(calling_convention.GetRegisterAt(0)), - index_location_, Primitive::kPrimInt); - arm64_codegen->MoveHelper(LocationFrom(calling_convention.GetRegisterAt(1)), - length_location_, Primitive::kPrimInt); - size_t offset = QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pThrowArrayBounds).SizeValue(); - __ Ldr(lr, MemOperand(tr, offset)); - __ Blr(lr); - codegen->RecordPcInfo(instruction_, instruction_->GetDexPc()); + __ Mov(calling_convention.GetRegisterAt(0).W(), cls_->GetTypeIndex()); + arm64_codegen->LoadCurrentMethod(calling_convention.GetRegisterAt(1).W()); + int32_t entry_point_offset = do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage) + : QUICK_ENTRY_POINT(pInitializeType); + arm64_codegen->InvokeRuntime(entry_point_offset, at_, dex_pc_); + + // Move the class to the desired location. + Location out = locations->Out(); + if (out.IsValid()) { + DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); + Primitive::Type type = at_->GetType(); + arm64_codegen->MoveHelper(out, calling_convention.GetReturnLocation(type), type); + } + + codegen->RestoreLiveRegisters(locations); + __ B(GetExitLabel()); } private: - HBoundsCheck* const instruction_; - const Location index_location_; - const Location length_location_; + // The class this slow path will load. + HLoadClass* const cls_; - DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM64); + // The instruction where this slow path is happening. + // (Might be the load class or an initialization check). + HInstruction* const at_; + + // The dex PC of `at_`. + const uint32_t dex_pc_; + + // Whether to initialize the class. + const bool do_clinit_; + + DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM64); +}; + +class LoadStringSlowPathARM64 : public SlowPathCodeARM64 { + public: + explicit LoadStringSlowPathARM64(HLoadString* instruction) : instruction_(instruction) {} + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = instruction_->GetLocations(); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); + CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); + + __ Bind(GetEntryLabel()); + codegen->SaveLiveRegisters(locations); + + InvokeRuntimeCallingConvention calling_convention; + arm64_codegen->LoadCurrentMethod(calling_convention.GetRegisterAt(0).W()); + __ Mov(calling_convention.GetRegisterAt(1).W(), instruction_->GetStringIndex()); + arm64_codegen->InvokeRuntime( + QUICK_ENTRY_POINT(pResolveString), instruction_, instruction_->GetDexPc()); + Primitive::Type type = instruction_->GetType(); + arm64_codegen->MoveHelper(locations->Out(), calling_convention.GetReturnLocation(type), type); + + codegen->RestoreLiveRegisters(locations); + __ B(GetExitLabel()); + } + + private: + HLoadString* const instruction_; + + DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM64); }; class NullCheckSlowPathARM64 : public SlowPathCodeARM64 { public: explicit NullCheckSlowPathARM64(HNullCheck* instr) : instruction_(instr) {} - virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); __ Bind(GetEntryLabel()); - int32_t offset = QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pThrowNullPointer).Int32Value(); - __ Ldr(lr, MemOperand(tr, offset)); - __ Blr(lr); - codegen->RecordPcInfo(instruction_, instruction_->GetDexPc()); + arm64_codegen->InvokeRuntime( + QUICK_ENTRY_POINT(pThrowNullPointer), instruction_, instruction_->GetDexPc()); } private: @@ -298,13 +399,18 @@ class SuspendCheckSlowPathARM64 : public SlowPathCodeARM64 { HBasicBlock* successor) : instruction_(instruction), successor_(successor) {} - virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - size_t offset = QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pTestSuspend).SizeValue(); + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); __ Bind(GetEntryLabel()); - __ Ldr(lr, MemOperand(tr, offset)); - __ Blr(lr); - codegen->RecordPcInfo(instruction_, instruction_->GetDexPc()); - __ B(GetReturnLabel()); + codegen->SaveLiveRegisters(instruction_->GetLocations()); + arm64_codegen->InvokeRuntime( + QUICK_ENTRY_POINT(pTestSuspend), instruction_, instruction_->GetDexPc()); + codegen->RestoreLiveRegisters(instruction_->GetLocations()); + if (successor_ == nullptr) { + __ B(GetReturnLabel()); + } else { + __ B(arm64_codegen->GetLabelOf(successor_)); + } } vixl::Label* GetReturnLabel() { @@ -324,6 +430,20 @@ class SuspendCheckSlowPathARM64 : public SlowPathCodeARM64 { DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM64); }; +class TypeCheckSlowPathARM64 : public SlowPathCodeARM64 { + public: + TypeCheckSlowPathARM64() {} + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + __ Bind(GetEntryLabel()); + __ Brk(__LINE__); // TODO: Unimplemented TypeCheckSlowPathARM64. + __ b(GetExitLabel()); + } + + private: + DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM64); +}; + #undef __ Location InvokeDexCallingConventionVisitor::GetNextLocation(Primitive::Type type) { @@ -356,11 +476,12 @@ CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph) location_builder_(graph, this), instruction_visitor_(graph, this) {} -#define __ reinterpret_cast<Arm64Assembler*>(GetAssembler())->vixl_masm_-> +#undef __ +#define __ GetVIXLAssembler()-> void CodeGeneratorARM64::GenerateFrameEntry() { // TODO: Add proper support for the stack overflow check. - UseScratchRegisterScope temps(assembler_.vixl_masm_); + UseScratchRegisterScope temps(GetVIXLAssembler()); Register temp = temps.AcquireX(); __ Add(temp, sp, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm64))); __ Ldr(temp, MemOperand(temp, 0)); @@ -378,7 +499,7 @@ void CodeGeneratorARM64::GenerateFrameEntry() { // ... : other preserved registers. // sp[frame_size - regs_size]: first preserved register. // ... : reserved frame space. - // sp[0] : context pointer. + // sp[0] : current method. } void CodeGeneratorARM64::GenerateFrameExit() { @@ -413,7 +534,7 @@ void CodeGeneratorARM64::Move(HInstruction* instruction, __ Mov(dst, value); } else { DCHECK(location.IsStackSlot() || location.IsDoubleStackSlot()); - UseScratchRegisterScope temps(assembler_.vixl_masm_); + UseScratchRegisterScope temps(GetVIXLAssembler()); Register temp = instruction->IsIntConstant() ? temps.AcquireW() : temps.AcquireX(); __ Mov(temp, value); __ Str(temp, StackOperandFrom(location)); @@ -465,7 +586,7 @@ Location CodeGeneratorARM64::GetStackLocation(HLoadLocal* load) const { } void CodeGeneratorARM64::MarkGCCard(Register object, Register value) { - UseScratchRegisterScope temps(assembler_.vixl_masm_); + UseScratchRegisterScope temps(GetVIXLAssembler()); Register card = temps.AcquireX(); Register temp = temps.AcquireX(); vixl::Label done; @@ -522,6 +643,19 @@ void CodeGeneratorARM64::DumpFloatingPointRegister(std::ostream& stream, int reg stream << Arm64ManagedRegister::FromDRegister(DRegister(reg)); } +void CodeGeneratorARM64::MoveConstant(CPURegister destination, HConstant* constant) { + if (constant->IsIntConstant() || constant->IsLongConstant()) { + __ Mov(Register(destination), + constant->IsIntConstant() ? constant->AsIntConstant()->GetValue() + : constant->AsLongConstant()->GetValue()); + } else if (constant->IsFloatConstant()) { + __ Fmov(FPRegister(destination), constant->AsFloatConstant()->GetValue()); + } else { + DCHECK(constant->IsDoubleConstant()); + __ Fmov(FPRegister(destination), constant->AsDoubleConstant()->GetValue()); + } +} + void CodeGeneratorARM64::MoveHelper(Location destination, Location source, Primitive::Type type) { @@ -544,13 +678,7 @@ void CodeGeneratorARM64::MoveHelper(Location destination, } else if (source.IsFpuRegister()) { __ Fmov(dst, FPRegisterFrom(source, type)); } else { - HConstant* cst = source.GetConstant(); - if (cst->IsFloatConstant()) { - __ Fmov(dst, cst->AsFloatConstant()->GetValue()); - } else { - DCHECK(cst->IsDoubleConstant()); - __ Fmov(dst, cst->AsDoubleConstant()->GetValue()); - } + MoveConstant(dst, source.GetConstant()); } } else { DCHECK(destination.IsStackSlot() || destination.IsDoubleStackSlot()); @@ -558,8 +686,21 @@ void CodeGeneratorARM64::MoveHelper(Location destination, __ Str(RegisterFrom(source, type), StackOperandFrom(destination)); } else if (source.IsFpuRegister()) { __ Str(FPRegisterFrom(source, type), StackOperandFrom(destination)); + } else if (source.IsConstant()) { + UseScratchRegisterScope temps(GetVIXLAssembler()); + HConstant* cst = source.GetConstant(); + CPURegister temp; + if (cst->IsIntConstant() || cst->IsLongConstant()) { + temp = cst->IsIntConstant() ? temps.AcquireW() : temps.AcquireX(); + } else { + DCHECK(cst->IsFloatConstant() || cst->IsDoubleConstant()); + temp = cst->IsFloatConstant() ? temps.AcquireS() : temps.AcquireD(); + } + MoveConstant(temp, cst); + __ Str(temp, StackOperandFrom(destination)); } else { - UseScratchRegisterScope temps(assembler_.vixl_masm_); + DCHECK(source.IsStackSlot() || source.IsDoubleStackSlot()); + UseScratchRegisterScope temps(GetVIXLAssembler()); Register temp = destination.IsDoubleStackSlot() ? temps.AcquireX() : temps.AcquireW(); __ Ldr(temp, StackOperandFrom(source)); __ Str(temp, StackOperandFrom(destination)); @@ -568,61 +709,89 @@ void CodeGeneratorARM64::MoveHelper(Location destination, } void CodeGeneratorARM64::Load(Primitive::Type type, - vixl::Register dst, + vixl::CPURegister dst, const vixl::MemOperand& src) { switch (type) { case Primitive::kPrimBoolean: - __ Ldrb(dst, src); + __ Ldrb(Register(dst), src); break; case Primitive::kPrimByte: - __ Ldrsb(dst, src); + __ Ldrsb(Register(dst), src); break; case Primitive::kPrimShort: - __ Ldrsh(dst, src); + __ Ldrsh(Register(dst), src); break; case Primitive::kPrimChar: - __ Ldrh(dst, src); + __ Ldrh(Register(dst), src); break; case Primitive::kPrimInt: case Primitive::kPrimNot: case Primitive::kPrimLong: - DCHECK(dst.Is64Bits() == (type == Primitive::kPrimLong)); - __ Ldr(dst, src); - break; case Primitive::kPrimFloat: case Primitive::kPrimDouble: + DCHECK(dst.Is64Bits() == Is64BitType(type)); + __ Ldr(dst, src); + break; case Primitive::kPrimVoid: LOG(FATAL) << "Unreachable type " << type; } } void CodeGeneratorARM64::Store(Primitive::Type type, - vixl::Register rt, + vixl::CPURegister rt, const vixl::MemOperand& dst) { switch (type) { case Primitive::kPrimBoolean: case Primitive::kPrimByte: - __ Strb(rt, dst); + __ Strb(Register(rt), dst); break; case Primitive::kPrimChar: case Primitive::kPrimShort: - __ Strh(rt, dst); + __ Strh(Register(rt), dst); break; case Primitive::kPrimInt: case Primitive::kPrimNot: case Primitive::kPrimLong: - DCHECK(rt.Is64Bits() == (type == Primitive::kPrimLong)); - __ Str(rt, dst); - break; case Primitive::kPrimFloat: case Primitive::kPrimDouble: + DCHECK(rt.Is64Bits() == Is64BitType(type)); + __ Str(rt, dst); + break; case Primitive::kPrimVoid: LOG(FATAL) << "Unreachable type " << type; } } -#undef __ -#define __ GetAssembler()->vixl_masm_-> +void CodeGeneratorARM64::LoadCurrentMethod(vixl::Register current_method) { + DCHECK(current_method.IsW()); + __ Ldr(current_method, MemOperand(sp, kCurrentMethodStackOffset)); +} + +void CodeGeneratorARM64::InvokeRuntime(int32_t entry_point_offset, + HInstruction* instruction, + uint32_t dex_pc) { + __ Ldr(lr, MemOperand(tr, entry_point_offset)); + __ Blr(lr); + RecordPcInfo(instruction, dex_pc); + DCHECK(instruction->IsSuspendCheck() + || instruction->IsBoundsCheck() + || instruction->IsNullCheck() + || instruction->IsDivZeroCheck() + || !IsLeafMethod()); +} + +void InstructionCodeGeneratorARM64::GenerateClassInitializationCheck(SlowPathCodeARM64* slow_path, + vixl::Register class_reg) { + UseScratchRegisterScope temps(GetVIXLAssembler()); + Register temp = temps.AcquireW(); + __ Ldr(temp, HeapOperand(class_reg, mirror::Class::StatusOffset())); + __ Cmp(temp, mirror::Class::kStatusInitialized); + __ B(lt, slow_path->GetEntryLabel()); + // Even if the initialized flag is set, we may be in a situation where caches are not synced + // properly. Therefore, we do a memory fence. + __ Dmb(InnerShareable, BarrierAll); + __ Bind(slow_path->GetExitLabel()); +} InstructionCodeGeneratorARM64::InstructionCodeGeneratorARM64(HGraph* graph, CodeGeneratorARM64* codegen) @@ -631,28 +800,14 @@ InstructionCodeGeneratorARM64::InstructionCodeGeneratorARM64(HGraph* graph, codegen_(codegen) {} #define FOR_EACH_UNIMPLEMENTED_INSTRUCTION(M) \ - M(And) \ - M(CheckCast) \ - M(ClinitCheck) \ - M(DivZeroCheck) \ - M(InstanceOf) \ - M(InvokeInterface) \ - M(LoadClass) \ - M(LoadException) \ - M(LoadString) \ - M(MonitorOperation) \ - M(Or) \ M(ParallelMove) \ - M(Rem) \ - M(StaticFieldGet) \ - M(StaticFieldSet) \ - M(Throw) \ - M(TypeConversion) \ - M(Xor) \ + M(Rem) #define UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name) name##UnimplementedInstructionBreakCode enum UnimplementedInstructionBreakCode { + // Using a base helps identify when we hit such breakpoints. + UnimplementedInstructionBreakCodeBaseCode = 0x900, #define ENUM_UNIMPLEMENTED_INSTRUCTION(name) UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name), FOR_EACH_UNIMPLEMENTED_INSTRUCTION(ENUM_UNIMPLEMENTED_INSTRUCTION) #undef ENUM_UNIMPLEMENTED_INSTRUCTION @@ -671,9 +826,9 @@ enum UnimplementedInstructionBreakCode { #undef DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS #undef UNIMPLEMENTED_INSTRUCTION_BREAK_CODE +#undef FOR_EACH_UNIMPLEMENTED_INSTRUCTION -void LocationsBuilderARM64::HandleAddSub(HBinaryOperation* instr) { - DCHECK(instr->IsAdd() || instr->IsSub()); +void LocationsBuilderARM64::HandleBinaryOp(HBinaryOperation* instr) { DCHECK_EQ(instr->InputCount(), 2U); LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr); Primitive::Type type = instr->GetResultType(); @@ -689,7 +844,7 @@ void LocationsBuilderARM64::HandleAddSub(HBinaryOperation* instr) { case Primitive::kPrimDouble: locations->SetInAt(0, Location::RequiresFpuRegister()); locations->SetInAt(1, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: @@ -697,9 +852,7 @@ void LocationsBuilderARM64::HandleAddSub(HBinaryOperation* instr) { } } -void InstructionCodeGeneratorARM64::HandleAddSub(HBinaryOperation* instr) { - DCHECK(instr->IsAdd() || instr->IsSub()); - +void InstructionCodeGeneratorARM64::HandleBinaryOp(HBinaryOperation* instr) { Primitive::Type type = instr->GetType(); switch (type) { @@ -710,8 +863,15 @@ void InstructionCodeGeneratorARM64::HandleAddSub(HBinaryOperation* instr) { Operand rhs = InputOperandAt(instr, 1); if (instr->IsAdd()) { __ Add(dst, lhs, rhs); - } else { + } else if (instr->IsAnd()) { + __ And(dst, lhs, rhs); + } else if (instr->IsOr()) { + __ Orr(dst, lhs, rhs); + } else if (instr->IsSub()) { __ Sub(dst, lhs, rhs); + } else { + DCHECK(instr->IsXor()); + __ Eor(dst, lhs, rhs); } break; } @@ -722,22 +882,32 @@ void InstructionCodeGeneratorARM64::HandleAddSub(HBinaryOperation* instr) { FPRegister rhs = InputFPRegisterAt(instr, 1); if (instr->IsAdd()) { __ Fadd(dst, lhs, rhs); - } else { + } else if (instr->IsSub()) { __ Fsub(dst, lhs, rhs); + } else { + LOG(FATAL) << "Unexpected floating-point binary operation"; } break; } default: - LOG(FATAL) << "Unexpected add/sub type " << type; + LOG(FATAL) << "Unexpected binary operation type " << type; } } void LocationsBuilderARM64::VisitAdd(HAdd* instruction) { - HandleAddSub(instruction); + HandleBinaryOp(instruction); } void InstructionCodeGeneratorARM64::VisitAdd(HAdd* instruction) { - HandleAddSub(instruction); + HandleBinaryOp(instruction); +} + +void LocationsBuilderARM64::VisitAnd(HAnd* instruction) { + HandleBinaryOp(instruction); +} + +void InstructionCodeGeneratorARM64::VisitAnd(HAnd* instruction) { + HandleBinaryOp(instruction); } void LocationsBuilderARM64::VisitArrayGet(HArrayGet* instruction) { @@ -752,11 +922,10 @@ void InstructionCodeGeneratorARM64::VisitArrayGet(HArrayGet* instruction) { LocationSummary* locations = instruction->GetLocations(); Primitive::Type type = instruction->GetType(); Register obj = InputRegisterAt(instruction, 0); - Register out = OutputRegister(instruction); Location index = locations->InAt(1); size_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(type)).Uint32Value(); MemOperand source(obj); - UseScratchRegisterScope temps(GetAssembler()->vixl_masm_); + UseScratchRegisterScope temps(GetVIXLAssembler()); if (index.IsConstant()) { offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(type); @@ -768,7 +937,7 @@ void InstructionCodeGeneratorARM64::VisitArrayGet(HArrayGet* instruction) { source = MemOperand(temp, offset); } - codegen_->Load(type, out, source); + codegen_->Load(type, OutputCPURegister(instruction), source); } void LocationsBuilderARM64::VisitArrayLength(HArrayLength* instruction) { @@ -802,18 +971,16 @@ void LocationsBuilderARM64::VisitArraySet(HArraySet* instruction) { void InstructionCodeGeneratorARM64::VisitArraySet(HArraySet* instruction) { Primitive::Type value_type = instruction->GetComponentType(); if (value_type == Primitive::kPrimNot) { - __ Ldr(lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAputObject).Int32Value())); - __ Blr(lr); - codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); - DCHECK(!codegen_->IsLeafMethod()); + codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject), instruction, instruction->GetDexPc()); + } else { LocationSummary* locations = instruction->GetLocations(); Register obj = InputRegisterAt(instruction, 0); - Register value = InputRegisterAt(instruction, 2); + CPURegister value = InputCPURegisterAt(instruction, 2); Location index = locations->InAt(1); size_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); MemOperand destination(obj); - UseScratchRegisterScope temps(GetAssembler()->vixl_masm_); + UseScratchRegisterScope temps(GetVIXLAssembler()); if (index.IsConstant()) { offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(value_type); @@ -829,6 +996,66 @@ void InstructionCodeGeneratorARM64::VisitArraySet(HArraySet* instruction) { } } +void LocationsBuilderARM64::VisitBoundsCheck(HBoundsCheck* instruction) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + if (instruction->HasUses()) { + locations->SetOut(Location::SameAsFirstInput()); + } +} + +void InstructionCodeGeneratorARM64::VisitBoundsCheck(HBoundsCheck* instruction) { + BoundsCheckSlowPathARM64* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM64(); + codegen_->AddSlowPath(slow_path); + + __ Cmp(InputRegisterAt(instruction, 0), InputOperandAt(instruction, 1)); + __ B(slow_path->GetEntryLabel(), hs); +} + +void LocationsBuilderARM64::VisitCheckCast(HCheckCast* instruction) { + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( + instruction, LocationSummary::kCallOnSlowPath); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); +} + +void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) { + UseScratchRegisterScope temps(GetVIXLAssembler()); + Register obj = InputRegisterAt(instruction, 0);; + Register cls = InputRegisterAt(instruction, 1);; + Register temp = temps.AcquireW(); + + SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM64(); + codegen_->AddSlowPath(slow_path); + + // TODO: avoid this check if we know obj is not null. + __ Cbz(obj, slow_path->GetExitLabel()); + // Compare the class of `obj` with `cls`. + __ Ldr(temp, HeapOperand(obj, mirror::Object::ClassOffset())); + __ Cmp(temp, cls); + __ B(ne, slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); +} + +void LocationsBuilderARM64::VisitClinitCheck(HClinitCheck* check) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath); + locations->SetInAt(0, Location::RequiresRegister()); + if (check->HasUses()) { + locations->SetOut(Location::SameAsFirstInput()); + } +} + +void InstructionCodeGeneratorARM64::VisitClinitCheck(HClinitCheck* check) { + // We assume the class is not null. + SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM64( + check->GetLoadClass(), check, check->GetDexPc(), true); + codegen_->AddSlowPath(slow_path); + GenerateClassInitializationCheck(slow_path, InputRegisterAt(check, 0)); +} + void LocationsBuilderARM64::VisitCompare(HCompare* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); @@ -847,7 +1074,7 @@ void InstructionCodeGeneratorARM64::VisitCompare(HCompare* instruction) { Register result = OutputRegister(instruction); Register left = InputRegisterAt(instruction, 0); Operand right = InputOperandAt(instruction, 1); - __ Subs(result, left, right); + __ Subs(result.X(), left, right); __ B(eq, &done); __ Mov(result, 1); __ Cneg(result, result, le); @@ -894,6 +1121,7 @@ void InstructionCodeGeneratorARM64::VisitCondition(HCondition* instruction) { void LocationsBuilderARM64::Visit##Name(H##Name* comp) { VisitCondition(comp); } \ void InstructionCodeGeneratorARM64::Visit##Name(H##Name* comp) { VisitCondition(comp); } FOR_EACH_CONDITION_INSTRUCTION(DEFINE_CONDITION_VISITORS) +#undef DEFINE_CONDITION_VISITORS #undef FOR_EACH_CONDITION_INSTRUCTION void LocationsBuilderARM64::VisitDiv(HDiv* div) { @@ -937,6 +1165,33 @@ void InstructionCodeGeneratorARM64::VisitDiv(HDiv* div) { } } +void LocationsBuilderARM64::VisitDivZeroCheck(HDivZeroCheck* instruction) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); + locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0))); + if (instruction->HasUses()) { + locations->SetOut(Location::SameAsFirstInput()); + } +} + +void InstructionCodeGeneratorARM64::VisitDivZeroCheck(HDivZeroCheck* instruction) { + SlowPathCodeARM64* slow_path = + new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM64(instruction); + codegen_->AddSlowPath(slow_path); + Location value = instruction->GetLocations()->InAt(0); + + if (value.IsConstant()) { + int64_t divisor = Int64ConstantFrom(value); + if (divisor == 0) { + __ B(slow_path->GetEntryLabel()); + } else { + LOG(FATAL) << "Divisions by non-null constants should have been optimized away."; + } + } else { + __ Cbz(InputRegisterAt(instruction, 0), slow_path->GetEntryLabel()); + } +} + void LocationsBuilderARM64::VisitDoubleConstant(HDoubleConstant* constant) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); @@ -956,7 +1211,7 @@ void InstructionCodeGeneratorARM64::VisitExit(HExit* exit) { UNUSED(exit); if (kIsDebugBuild) { down_cast<Arm64Assembler*>(GetAssembler())->Comment("Unreachable"); - __ Brk(0); // TODO: Introduce special markers for such code locations. + __ Brk(__LINE__); // TODO: Introduce special markers for such code locations. } } @@ -1039,7 +1294,7 @@ void LocationsBuilderARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction void InstructionCodeGeneratorARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { MemOperand field = MemOperand(InputRegisterAt(instruction, 0), instruction->GetFieldOffset().Uint32Value()); - codegen_->Load(instruction->GetType(), OutputRegister(instruction), field); + codegen_->Load(instruction->GetType(), OutputCPURegister(instruction), field); } void LocationsBuilderARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { @@ -1050,12 +1305,54 @@ void LocationsBuilderARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction void InstructionCodeGeneratorARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { Primitive::Type field_type = instruction->GetFieldType(); - Register value = InputRegisterAt(instruction, 1); + CPURegister value = InputCPURegisterAt(instruction, 1); Register obj = InputRegisterAt(instruction, 0); codegen_->Store(field_type, value, MemOperand(obj, instruction->GetFieldOffset().Uint32Value())); if (field_type == Primitive::kPrimNot) { - codegen_->MarkGCCard(obj, value); + codegen_->MarkGCCard(obj, Register(value)); + } +} + +void LocationsBuilderARM64::VisitInstanceOf(HInstanceOf* instruction) { + LocationSummary::CallKind call_kind = + instruction->IsClassFinal() ? LocationSummary::kNoCall : LocationSummary::kCallOnSlowPath; + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), true); // The output does overlap inputs. +} + +void InstructionCodeGeneratorARM64::VisitInstanceOf(HInstanceOf* instruction) { + LocationSummary* locations = instruction->GetLocations(); + Register obj = InputRegisterAt(instruction, 0);; + Register cls = InputRegisterAt(instruction, 1);; + Register out = OutputRegister(instruction); + + vixl::Label done; + + // Return 0 if `obj` is null. + // TODO: Avoid this check if we know `obj` is not null. + __ Mov(out, 0); + __ Cbz(obj, &done); + + // Compare the class of `obj` with `cls`. + __ Ldr(out, MemOperand(obj, mirror::Object::ClassOffset().Int32Value())); + __ Cmp(out, cls); + if (instruction->IsClassFinal()) { + // Classes must be equal for the instanceof to succeed. + __ Cset(out, eq); + } else { + // If the classes are not equal, we go into a slow path. + DCHECK(locations->OnlyCallsOnSlowPath()); + SlowPathCodeARM64* slow_path = + new (GetGraph()->GetArena()) TypeCheckSlowPathARM64(); + codegen_->AddSlowPath(slow_path); + __ B(ne, slow_path->GetEntryLabel()); + __ Mov(out, 1); + __ Bind(slow_path->GetExitLabel()); } + + __ Bind(&done); } void LocationsBuilderARM64::VisitIntConstant(HIntConstant* constant) { @@ -1068,14 +1365,6 @@ void InstructionCodeGeneratorARM64::VisitIntConstant(HIntConstant* constant) { UNUSED(constant); } -void LocationsBuilderARM64::VisitInvokeStatic(HInvokeStatic* invoke) { - HandleInvoke(invoke); -} - -void LocationsBuilderARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { - HandleInvoke(invoke); -} - void LocationsBuilderARM64::HandleInvoke(HInvoke* invoke) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(invoke, LocationSummary::kCall); @@ -1093,6 +1382,50 @@ void LocationsBuilderARM64::HandleInvoke(HInvoke* invoke) { } } +void LocationsBuilderARM64::VisitInvokeInterface(HInvokeInterface* invoke) { + HandleInvoke(invoke); +} + +void InstructionCodeGeneratorARM64::VisitInvokeInterface(HInvokeInterface* invoke) { + // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError. + Register temp = WRegisterFrom(invoke->GetLocations()->GetTemp(0)); + uint32_t method_offset = mirror::Class::EmbeddedImTableOffset().Uint32Value() + + (invoke->GetImtIndex() % mirror::Class::kImtSize) * sizeof(mirror::Class::ImTableEntry); + Location receiver = invoke->GetLocations()->InAt(0); + Offset class_offset = mirror::Object::ClassOffset(); + Offset entry_point = mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(); + + // The register ip1 is required to be used for the hidden argument in + // art_quick_imt_conflict_trampoline, so prevent VIXL from using it. + UseScratchRegisterScope scratch_scope(GetVIXLAssembler()); + scratch_scope.Exclude(ip1); + __ Mov(ip1, invoke->GetDexMethodIndex()); + + // temp = object->GetClass(); + if (receiver.IsStackSlot()) { + __ Ldr(temp, StackOperandFrom(receiver)); + __ Ldr(temp, HeapOperand(temp, class_offset)); + } else { + __ Ldr(temp, HeapOperandFrom(receiver, class_offset)); + } + // temp = temp->GetImtEntryAt(method_offset); + __ Ldr(temp, HeapOperand(temp, method_offset)); + // lr = temp->GetEntryPoint(); + __ Ldr(lr, HeapOperand(temp, entry_point)); + // lr(); + __ Blr(lr); + DCHECK(!codegen_->IsLeafMethod()); + codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); +} + +void LocationsBuilderARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { + HandleInvoke(invoke); +} + +void LocationsBuilderARM64::VisitInvokeStatic(HInvokeStatic* invoke) { + HandleInvoke(invoke); +} + void InstructionCodeGeneratorARM64::VisitInvokeStatic(HInvokeStatic* invoke) { Register temp = WRegisterFrom(invoke->GetLocations()->GetTemp(0)); // Make sure that ArtMethod* is passed in W0 as per the calling convention @@ -1108,7 +1441,7 @@ void InstructionCodeGeneratorARM64::VisitInvokeStatic(HInvokeStatic* invoke) { // Currently we implement the app -> app logic, which looks up in the resolve cache. // temp = method; - __ Ldr(temp, MemOperand(sp, kCurrentMethodStackOffset)); + codegen_->LoadCurrentMethod(temp); // temp = temp->dex_cache_resolved_methods_; __ Ldr(temp, MemOperand(temp.X(), mirror::ArtMethod::DexCacheResolvedMethodsOffset().SizeValue())); @@ -1139,8 +1472,7 @@ void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { __ Ldr(temp.W(), MemOperand(temp, class_offset.SizeValue())); } else { DCHECK(receiver.IsRegister()); - __ Ldr(temp.W(), HeapOperandFrom(receiver, Primitive::kPrimNot, - class_offset)); + __ Ldr(temp.W(), HeapOperandFrom(receiver, class_offset)); } // temp = temp->GetMethodAt(method_offset); __ Ldr(temp.W(), MemOperand(temp, method_offset)); @@ -1152,6 +1484,50 @@ void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } +void LocationsBuilderARM64::VisitLoadClass(HLoadClass* cls) { + LocationSummary::CallKind call_kind = cls->CanCallRuntime() ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall; + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); + locations->SetOut(Location::RequiresRegister()); +} + +void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) { + Register out = OutputRegister(cls); + if (cls->IsReferrersClass()) { + DCHECK(!cls->CanCallRuntime()); + DCHECK(!cls->MustGenerateClinitCheck()); + codegen_->LoadCurrentMethod(out); + __ Ldr(out, HeapOperand(out, mirror::ArtMethod::DeclaringClassOffset())); + } else { + DCHECK(cls->CanCallRuntime()); + codegen_->LoadCurrentMethod(out); + __ Ldr(out, HeapOperand(out, mirror::ArtMethod::DexCacheResolvedTypesOffset())); + __ Ldr(out, MemOperand(out.X(), CodeGenerator::GetCacheOffset(cls->GetTypeIndex()))); + + SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM64( + cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); + codegen_->AddSlowPath(slow_path); + __ Cbz(out, slow_path->GetEntryLabel()); + if (cls->MustGenerateClinitCheck()) { + GenerateClassInitializationCheck(slow_path, out); + } else { + __ Bind(slow_path->GetExitLabel()); + } + } +} + +void LocationsBuilderARM64::VisitLoadException(HLoadException* load) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall); + locations->SetOut(Location::RequiresRegister()); +} + +void InstructionCodeGeneratorARM64::VisitLoadException(HLoadException* instruction) { + MemOperand exception = MemOperand(tr, Thread::ExceptionOffset<kArm64WordSize>().Int32Value()); + __ Ldr(OutputRegister(instruction), exception); + __ Str(wzr, exception); +} + void LocationsBuilderARM64::VisitLoadLocal(HLoadLocal* load) { load->SetLocations(nullptr); } @@ -1161,6 +1537,24 @@ void InstructionCodeGeneratorARM64::VisitLoadLocal(HLoadLocal* load) { UNUSED(load); } +void LocationsBuilderARM64::VisitLoadString(HLoadString* load) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath); + locations->SetOut(Location::RequiresRegister()); +} + +void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) { + SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM64(load); + codegen_->AddSlowPath(slow_path); + + Register out = OutputRegister(load); + codegen_->LoadCurrentMethod(out); + __ Ldr(out, HeapOperand(out, mirror::ArtMethod::DexCacheStringsOffset())); + __ Ldr(out, MemOperand(out.X(), CodeGenerator::GetCacheOffset(load->GetStringIndex()))); + __ Cbz(out, slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); +} + void LocationsBuilderARM64::VisitLocal(HLocal* local) { local->SetLocations(nullptr); } @@ -1179,6 +1573,20 @@ void InstructionCodeGeneratorARM64::VisitLongConstant(HLongConstant* constant) { UNUSED(constant); } +void LocationsBuilderARM64::VisitMonitorOperation(HMonitorOperation* instruction) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall); + InvokeRuntimeCallingConvention calling_convention; + locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); +} + +void InstructionCodeGeneratorARM64::VisitMonitorOperation(HMonitorOperation* instruction) { + codegen_->InvokeRuntime(instruction->IsEnter() + ? QUICK_ENTRY_POINT(pLockObject) : QUICK_ENTRY_POINT(pUnlockObject), + instruction, + instruction->GetDexPc()); +} + void LocationsBuilderARM64::VisitMul(HMul* mul) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); @@ -1194,7 +1602,7 @@ void LocationsBuilderARM64::VisitMul(HMul* mul) { case Primitive::kPrimDouble: locations->SetInAt(0, Location::RequiresFpuRegister()); locations->SetInAt(1, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: @@ -1224,15 +1632,15 @@ void LocationsBuilderARM64::VisitNeg(HNeg* neg) { new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); switch (neg->GetResultType()) { case Primitive::kPrimInt: - case Primitive::kPrimLong: { + case Primitive::kPrimLong: locations->SetInAt(0, Location::RegisterOrConstant(neg->InputAt(0))); - locations->SetOut(Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); break; - } case Primitive::kPrimFloat: case Primitive::kPrimDouble: - LOG(FATAL) << "Not yet implemented neg type " << neg->GetResultType(); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: @@ -1249,7 +1657,7 @@ void InstructionCodeGeneratorARM64::VisitNeg(HNeg* neg) { case Primitive::kPrimFloat: case Primitive::kPrimDouble: - LOG(FATAL) << "Not yet implemented neg type " << neg->GetResultType(); + __ Fneg(OutputFPRegister(neg), InputFPRegisterAt(neg, 0)); break; default: @@ -1274,14 +1682,10 @@ void InstructionCodeGeneratorARM64::VisitNewArray(HNewArray* instruction) { DCHECK(type_index.Is(w0)); Register current_method = RegisterFrom(locations->GetTemp(1), Primitive::kPrimNot); DCHECK(current_method.Is(w1)); - __ Ldr(current_method, MemOperand(sp, kCurrentMethodStackOffset)); + codegen_->LoadCurrentMethod(current_method); __ Mov(type_index, instruction->GetTypeIndex()); - int32_t quick_entrypoint_offset = - QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocArrayWithAccessCheck).Int32Value(); - __ Ldr(lr, MemOperand(tr, quick_entrypoint_offset)); - __ Blr(lr); - codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); - DCHECK(!codegen_->IsLeafMethod()); + codegen_->InvokeRuntime( + QUICK_ENTRY_POINT(pAllocArrayWithAccessCheck), instruction, instruction->GetDexPc()); } void LocationsBuilderARM64::VisitNewInstance(HNewInstance* instruction) { @@ -1299,14 +1703,10 @@ void InstructionCodeGeneratorARM64::VisitNewInstance(HNewInstance* instruction) DCHECK(type_index.Is(w0)); Register current_method = RegisterFrom(locations->GetTemp(1), Primitive::kPrimNot); DCHECK(current_method.Is(w1)); - __ Ldr(current_method, MemOperand(sp, kCurrentMethodStackOffset)); + codegen_->LoadCurrentMethod(current_method); __ Mov(type_index, instruction->GetTypeIndex()); - int32_t quick_entrypoint_offset = - QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocObjectWithAccessCheck).Int32Value(); - __ Ldr(lr, MemOperand(tr, quick_entrypoint_offset)); - __ Blr(lr); - codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); - DCHECK(!codegen_->IsLeafMethod()); + codegen_->InvokeRuntime( + QUICK_ENTRY_POINT(pAllocObjectWithAccessCheck), instruction, instruction->GetDexPc()); } void LocationsBuilderARM64::VisitNot(HNot* instruction) { @@ -1355,6 +1755,14 @@ void InstructionCodeGeneratorARM64::VisitNullCheck(HNullCheck* instruction) { } } +void LocationsBuilderARM64::VisitOr(HOr* instruction) { + HandleBinaryOp(instruction); +} + +void InstructionCodeGeneratorARM64::VisitOr(HOr* instruction) { + HandleBinaryOp(instruction); +} + void LocationsBuilderARM64::VisitParameterValue(HParameterValue* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); @@ -1435,31 +1843,43 @@ void InstructionCodeGeneratorARM64::VisitStoreLocal(HStoreLocal* store) { } void LocationsBuilderARM64::VisitSub(HSub* instruction) { - HandleAddSub(instruction); + HandleBinaryOp(instruction); } void InstructionCodeGeneratorARM64::VisitSub(HSub* instruction) { - HandleAddSub(instruction); + HandleBinaryOp(instruction); } -void LocationsBuilderARM64::VisitBoundsCheck(HBoundsCheck* instruction) { +void LocationsBuilderARM64::VisitStaticFieldGet(HStaticFieldGet* instruction) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); +} + +void InstructionCodeGeneratorARM64::VisitStaticFieldGet(HStaticFieldGet* instruction) { + Register cls = InputRegisterAt(instruction, 0); + uint32_t offset = instruction->GetFieldOffset().Uint32Value(); + codegen_->Load(instruction->GetType(), OutputCPURegister(instruction), MemOperand(cls, offset)); +} + +void LocationsBuilderARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::RequiresRegister()); - if (instruction->HasUses()) { - locations->SetOut(Location::SameAsFirstInput()); - } } -void InstructionCodeGeneratorARM64::VisitBoundsCheck(HBoundsCheck* instruction) { - LocationSummary* locations = instruction->GetLocations(); - BoundsCheckSlowPathARM64* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM64( - instruction, locations->InAt(0), locations->InAt(1)); - codegen_->AddSlowPath(slow_path); +void InstructionCodeGeneratorARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) { + CPURegister value = InputCPURegisterAt(instruction, 1); + Register cls = InputRegisterAt(instruction, 0); + uint32_t offset = instruction->GetFieldOffset().Uint32Value(); + Primitive::Type field_type = instruction->GetFieldType(); - __ Cmp(InputRegisterAt(instruction, 0), InputOperandAt(instruction, 1)); - __ B(slow_path->GetEntryLabel(), hs); + codegen_->Store(field_type, value, MemOperand(cls, offset)); + if (field_type == Primitive::kPrimNot) { + codegen_->MarkGCCard(cls, Register(value)); + } } void LocationsBuilderARM64::VisitSuspendCheck(HSuspendCheck* instruction) { @@ -1486,5 +1906,74 @@ void InstructionCodeGeneratorARM64::VisitTemporary(HTemporary* temp) { UNUSED(temp); } +void LocationsBuilderARM64::VisitThrow(HThrow* instruction) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall); + InvokeRuntimeCallingConvention calling_convention; + locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); +} + +void InstructionCodeGeneratorARM64::VisitThrow(HThrow* instruction) { + codegen_->InvokeRuntime( + QUICK_ENTRY_POINT(pDeliverException), instruction, instruction->GetDexPc()); +} + +void LocationsBuilderARM64::VisitTypeConversion(HTypeConversion* conversion) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(conversion, LocationSummary::kNoCall); + Primitive::Type input_type = conversion->GetInputType(); + Primitive::Type result_type = conversion->GetResultType(); + if ((input_type == Primitive::kPrimNot) || (input_type == Primitive::kPrimVoid) || + (result_type == Primitive::kPrimNot) || (result_type == Primitive::kPrimVoid)) { + LOG(FATAL) << "Unexpected type conversion from " << input_type << " to " << result_type; + } + + if (IsFPType(input_type)) { + locations->SetInAt(0, Location::RequiresFpuRegister()); + } else { + locations->SetInAt(0, Location::RequiresRegister()); + } + + if (IsFPType(result_type)) { + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + } else { + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + } +} + +void InstructionCodeGeneratorARM64::VisitTypeConversion(HTypeConversion* conversion) { + Primitive::Type result_type = conversion->GetResultType(); + Primitive::Type input_type = conversion->GetInputType(); + + DCHECK_NE(input_type, result_type); + + if (IsIntegralType(result_type) && IsIntegralType(input_type)) { + int result_size = Primitive::ComponentSize(result_type); + int input_size = Primitive::ComponentSize(input_type); + int min_size = kBitsPerByte * std::min(result_size, input_size); + if ((result_type == Primitive::kPrimChar) || + ((input_type == Primitive::kPrimChar) && (result_size > input_size))) { + __ Ubfx(OutputRegister(conversion), InputRegisterAt(conversion, 0), 0, min_size); + } else { + __ Sbfx(OutputRegister(conversion), InputRegisterAt(conversion, 0), 0, min_size); + } + return; + } + + LOG(FATAL) << "Unexpected or unimplemented type conversion from " << input_type + << " to " << result_type; +} + +void LocationsBuilderARM64::VisitXor(HXor* instruction) { + HandleBinaryOp(instruction); +} + +void InstructionCodeGeneratorARM64::VisitXor(HXor* instruction) { + HandleBinaryOp(instruction); +} + +#undef __ +#undef QUICK_ENTRY_POINT + } // namespace arm64 } // namespace art diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h index 54e87f4d9c..6b71b94532 100644 --- a/compiler/optimizing/code_generator_arm64.h +++ b/compiler/optimizing/code_generator_arm64.h @@ -29,6 +29,7 @@ namespace art { namespace arm64 { class CodeGeneratorARM64; +class SlowPathCodeARM64; static constexpr size_t kArm64WordSize = 8; static const vixl::Register kParameterCoreRegisters[] = { @@ -103,9 +104,11 @@ class InstructionCodeGeneratorARM64 : public HGraphVisitor { void LoadCurrentMethod(XRegister reg); Arm64Assembler* GetAssembler() const { return assembler_; } + vixl::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->vixl_masm_; } private: - void HandleAddSub(HBinaryOperation* instr); + void GenerateClassInitializationCheck(SlowPathCodeARM64* slow_path, vixl::Register class_reg); + void HandleBinaryOp(HBinaryOperation* instr); Arm64Assembler* const assembler_; CodeGeneratorARM64* const codegen_; @@ -124,7 +127,7 @@ class LocationsBuilderARM64 : public HGraphVisitor { #undef DECLARE_VISIT_INSTRUCTION private: - void HandleAddSub(HBinaryOperation* instr); + void HandleBinaryOp(HBinaryOperation* instr); void HandleInvoke(HInvoke* instr); CodeGeneratorARM64* const codegen_; @@ -162,9 +165,10 @@ class CodeGeneratorARM64 : public CodeGenerator { return kArm64WordSize; } - uintptr_t GetAddressOf(HBasicBlock* block ATTRIBUTE_UNUSED) const OVERRIDE { - UNIMPLEMENTED(INFO) << "TODO: GetAddressOf"; - return 0u; + uintptr_t GetAddressOf(HBasicBlock* block) const OVERRIDE { + vixl::Label* block_entry_label = GetLabelOf(block); + DCHECK(block_entry_label->IsBound()); + return block_entry_label->location(); } size_t FrameEntrySpillSize() const OVERRIDE; @@ -172,6 +176,7 @@ class CodeGeneratorARM64 : public CodeGenerator { HGraphVisitor* GetLocationBuilder() OVERRIDE { return &location_builder_; } HGraphVisitor* GetInstructionVisitor() OVERRIDE { return &instruction_visitor_; } Arm64Assembler* GetAssembler() OVERRIDE { return &assembler_; } + vixl::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->vixl_masm_; } // Emit a write barrier. void MarkGCCard(vixl::Register object, vixl::Register value); @@ -185,18 +190,18 @@ class CodeGeneratorARM64 : public CodeGenerator { Location GetStackLocation(HLoadLocal* load) const OVERRIDE; - size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE { + size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) { UNUSED(stack_index); UNUSED(reg_id); - UNIMPLEMENTED(INFO) << "TODO: SaveCoreRegister"; - return 0; + LOG(INFO) << "CodeGeneratorARM64::SaveCoreRegister()"; + return kArm64WordSize; } - size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE { + size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { UNUSED(stack_index); UNUSED(reg_id); - UNIMPLEMENTED(INFO) << "TODO: RestoreCoreRegister"; - return 0; + LOG(INFO) << "CodeGeneratorARM64::RestoreCoreRegister()"; + return kArm64WordSize; } // The number of registers that can be allocated. The register allocator may @@ -226,9 +231,14 @@ class CodeGeneratorARM64 : public CodeGenerator { } // Code generation helpers. + void MoveConstant(vixl::CPURegister destination, HConstant* constant); void MoveHelper(Location destination, Location source, Primitive::Type type); - void Load(Primitive::Type type, vixl::Register dst, const vixl::MemOperand& src); - void Store(Primitive::Type type, vixl::Register rt, const vixl::MemOperand& dst); + void Load(Primitive::Type type, vixl::CPURegister dst, const vixl::MemOperand& src); + void Store(Primitive::Type type, vixl::CPURegister rt, const vixl::MemOperand& dst); + void LoadCurrentMethod(vixl::Register current_method); + + // Generate code to invoke a runtime entry point. + void InvokeRuntime(int32_t offset, HInstruction* instruction, uint32_t dex_pc); ParallelMoveResolver* GetMoveResolver() OVERRIDE { UNIMPLEMENTED(INFO) << "TODO: MoveResolver"; diff --git a/test/Android.run-test.mk b/test/Android.run-test.mk index 0f58234a34..8749f41ab1 100644 --- a/test/Android.run-test.mk +++ b/test/Android.run-test.mk @@ -307,164 +307,38 @@ TEST_ART_BROKEN_DEFAULT_RUN_TESTS := # Known broken tests for the arm64 optimizing compiler backend. TEST_ART_BROKEN_OPTIMIZING_ARM64_RUN_TESTS := \ - 001-HelloWorld \ - 002-sleep \ 003-omnibus-opcodes \ - 004-InterfaceTest \ - 004-JniTest \ 004-NativeAllocations \ 004-ReferenceMap \ - 004-SignalTest \ - 004-StackWalk \ - 004-UnsafeTest \ 005-annotations \ - 006-args \ - 007-count10 \ - 008-exceptions \ 009-instanceof \ 010-instance \ - 011-array-copy \ - 013-math2 \ - 014-math3 \ - 016-intern \ - 017-float \ - 018-stack-overflow \ - 019-wrong-array-type \ - 020-string \ - 021-string2 \ - 022-interface \ + 012-math \ 023-many-interfaces \ - 024-illegal-access \ - 025-access-controller \ - 026-access \ - 028-array-write \ - 029-assert \ - 030-bad-finalizer \ - 031-class-attributes \ - 032-concrete-sub \ - 033-class-init-deadlock \ - 034-call-null \ - 035-enum \ - 036-finalizer \ 037-inherit \ - 038-inner-null \ - 039-join-main \ - 040-miranda \ - 042-new-instance \ - 043-privates \ + 047-inherit \ 044-proxy \ 045-reflect-array \ 046-reflect \ 047-returns \ - 049-show-object \ - 050-sync-test \ - 051-thread \ - 052-verifier-fun \ - 054-uncaught \ - 055-enum-performance \ - 056-const-string-jumbo \ - 058-enum-order \ - 061-out-of-memory \ 062-character-encodings \ 063-process-manager \ - 064-field-access \ - 065-mismatched-implements \ - 066-mismatched-super \ - 067-preemptive-unpark \ 068-classloader \ 069-field-type \ - 070-nio-buffer \ 071-dexfile \ - 072-precise-gc \ - 074-gc-thrash \ - 075-verification-error \ - 076-boolean-put \ - 077-method-override \ - 078-polymorphic-virtual \ - 079-phantom \ - 080-oom-throw \ - 081-hot-exceptions \ - 082-inline-execute \ 083-compiler-regressions \ - 084-class-init \ - 085-old-style-inner-class \ - 086-null-super \ - 087-gc-after-link \ - 088-monitor-verification \ - 090-loop-formation \ - 092-locale \ - 093-serialization \ - 094-pattern \ - 096-array-copy-concurrent-gc \ - 097-duplicate-method \ - 098-ddmc \ - 100-reflect2 \ - 101-fibonacci \ - 102-concurrent-gc \ - 103-string-append \ - 104-growth-limit \ - 105-invoke \ 106-exceptions2 \ 107-int-math2 \ - 108-check-cast \ - 109-suspend-check \ - 110-field-access \ - 111-unresolvable-exception \ - 112-double-math \ - 113-multidex \ 114-ParallelGC \ - 117-nopatchoat \ - 118-noimage-dex2oat \ - 119-noimage-patchoat \ - 120-hashcode \ - 121-modifiers \ - 121-simple-suspend-check \ - 122-npe \ - 123-compiler-regressions-mt \ - 124-missing-classes \ - 125-gc-and-classloading \ - 126-miranda-multidex \ 201-built-in-exception-detail-messages \ - 202-thread-oome \ - 300-package-override \ - 301-abstract-protected \ - 303-verification-stress \ - 304-method-tracing \ - 401-optimizing-compiler \ - 402-optimizing-control-flow \ - 403-optimizing-long \ - 404-optimizing-allocator \ - 405-optimizing-long-allocator \ - 406-fields \ 407-arrays \ - 409-materialized-condition \ - 410-floats \ - 411-optimizing-arith \ 412-new-array \ - 413-regalloc-regression \ - 414-optimizing-arith-sub \ - 414-static-fields \ - 415-optimizing-arith-neg \ - 416-optimizing-arith-not \ - 417-optimizing-arith-div \ - 418-const-string \ - 419-long-parameter \ - 420-const-class \ - 421-exceptions \ - 421-large-frame \ 422-instanceof \ 422-type-conversion \ - 423-invoke-interface \ 424-checkcast \ - 426-monitor \ - 427-bitwise \ 427-bounds \ 428-optimizing-arith-rem \ - 700-LoadArgRegs \ 701-easy-div-rem \ - 702-LargeBranchOffset \ - 703-floating-point-div \ - 800-smali ifneq (,$(filter optimizing,$(COMPILER_TYPES))) ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,target,$(RUN_TYPES),$(PREBUILD_TYPES), \ |