AArch64: Use cbnz instead of cmp/b.ne pair for atomic operations.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@176253 91177308-0d34-0410-b5e6-96231b3b80d8

3 files changed, 61 insertions, 111 deletions

diff --git a/lib/Target/AArch64/AArch64ISelLowering.cpp b/lib/Target/AArch64/AArch64ISelLowering.cpp
index cea7f918df..4981fbaf71 100644
--- a/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -341,8 +341,7 @@ AArch64TargetLowering::emitAtomicBinary(MachineInstr *MI, MachineBasicBlock *BB,
   //   ldxr dest, ptr
   //   <binop> scratch, dest, incr
   //   stxr stxr_status, scratch, ptr
-  //   cmp stxr_status, #0
-  //   b.ne loopMBB
+  //   cbnz stxr_status, loopMBB
   //   fallthrough --> exitMBB
   BB = loopMBB;
   BuildMI(BB, dl, TII->get(ldrOpc), dest).addReg(ptr);
@@ -364,10 +363,8 @@ AArch64TargetLowering::emitAtomicBinary(MachineInstr *MI, MachineBasicBlock *BB,
   MRI.constrainRegClass(stxr_status, &AArch64::GPR32wspRegClass);
 
   BuildMI(BB, dl, TII->get(strOpc), stxr_status).addReg(scratch).addReg(ptr);
-  BuildMI(BB, dl, TII->get(AArch64::SUBwwi_lsl0_cmp))
-    .addReg(stxr_status).addImm(0);
-  BuildMI(BB, dl, TII->get(AArch64::Bcc))
-    .addImm(A64CC::NE).addMBB(loopMBB);
+  BuildMI(BB, dl, TII->get(AArch64::CBNZw))
+    .addReg(stxr_status).addMBB(loopMBB);
 
   BB->addSuccessor(loopMBB);
   BB->addSuccessor(exitMBB);
@@ -437,8 +434,7 @@ AArch64TargetLowering::emitAtomicBinaryMinMax(MachineInstr *MI,
   //   cmp incr, dest (, sign extend if necessary)
   //   csel scratch, dest, incr, cond
   //   stxr stxr_status, scratch, ptr
-  //   cmp stxr_status, #0
-  //   b.ne loopMBB
+  //   cbnz stxr_status, loopMBB
   //   fallthrough --> exitMBB
   BB = loopMBB;
   BuildMI(BB, dl, TII->get(ldrOpc), dest).addReg(ptr);
@@ -457,10 +453,8 @@ AArch64TargetLowering::emitAtomicBinaryMinMax(MachineInstr *MI,
 
   BuildMI(BB, dl, TII->get(strOpc), stxr_status)
     .addReg(scratch).addReg(ptr);
-  BuildMI(BB, dl, TII->get(AArch64::SUBwwi_lsl0_cmp))
-    .addReg(stxr_status).addImm(0);
-  BuildMI(BB, dl, TII->get(AArch64::Bcc))
-    .addImm(A64CC::NE).addMBB(loopMBB);
+  BuildMI(BB, dl, TII->get(AArch64::CBNZw))
+    .addReg(stxr_status).addMBB(loopMBB);
 
   BB->addSuccessor(loopMBB);
   BB->addSuccessor(exitMBB);
@@ -533,17 +527,14 @@ AArch64TargetLowering::emitAtomicCmpSwap(MachineInstr *MI,
 
   // loop2MBB:
   //   strex stxr_status, newval, [ptr]
-  //   cmp stxr_status, #0
-  //   b.ne loop1MBB
+  //   cbnz stxr_status, loop1MBB
   BB = loop2MBB;
   unsigned stxr_status = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
   MRI.constrainRegClass(stxr_status, &AArch64::GPR32wspRegClass);
 
   BuildMI(BB, dl, TII->get(strOpc), stxr_status).addReg(newval).addReg(ptr);
-  BuildMI(BB, dl, TII->get(AArch64::SUBwwi_lsl0_cmp))
-    .addReg(stxr_status).addImm(0);
-  BuildMI(BB, dl, TII->get(AArch64::Bcc))
-    .addImm(A64CC::NE).addMBB(loop1MBB);
+  BuildMI(BB, dl, TII->get(AArch64::CBNZw))
+    .addReg(stxr_status).addMBB(loop1MBB);
   BB->addSuccessor(loop1MBB);
   BB->addSuccessor(exitMBB);
 
diff --git a/lib/Target/AArch64/AArch64InstrInfo.td b/lib/Target/AArch64/AArch64InstrInfo.td
index 562a7f60bb..78c4ad16d7 100644
--- a/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/lib/Target/AArch64/AArch64InstrInfo.td
@@ -159,7 +159,7 @@ let Defs = [XSP], Uses = [XSP] in {
 // Atomic operation pseudo-instructions
 //===----------------------------------------------------------------------===//
 
-let usesCustomInserter = 1, Defs = [NZCV] in {
+let usesCustomInserter = 1 in {
 multiclass AtomicSizes<string opname> {
   def _I8 : PseudoInst<(outs GPR32:$dst), (ins GPR64:$ptr, GPR32:$incr),
     [(set GPR32:$dst, (!cast<SDNode>(opname # "_8") GPR64:$ptr, GPR32:$incr))]>;
@@ -178,11 +178,14 @@ defm ATOMIC_LOAD_AND  : AtomicSizes<"atomic_load_and">;
 defm ATOMIC_LOAD_OR   : AtomicSizes<"atomic_load_or">;
 defm ATOMIC_LOAD_XOR  : AtomicSizes<"atomic_load_xor">;
 defm ATOMIC_LOAD_NAND : AtomicSizes<"atomic_load_nand">;
-defm ATOMIC_LOAD_MIN  : AtomicSizes<"atomic_load_min">;
-defm ATOMIC_LOAD_MAX  : AtomicSizes<"atomic_load_max">;
-defm ATOMIC_LOAD_UMIN : AtomicSizes<"atomic_load_umin">;
-defm ATOMIC_LOAD_UMAX : AtomicSizes<"atomic_load_umax">;
 defm ATOMIC_SWAP      : AtomicSizes<"atomic_swap">;
+let Defs = [NZCV] in {
+  // These operations need a CMP to calculate the correct value
+  defm ATOMIC_LOAD_MIN  : AtomicSizes<"atomic_load_min">;
+  defm ATOMIC_LOAD_MAX  : AtomicSizes<"atomic_load_max">;
+  defm ATOMIC_LOAD_UMIN : AtomicSizes<"atomic_load_umin">;
+  defm ATOMIC_LOAD_UMAX : AtomicSizes<"atomic_load_umax">;
+}
 
 let usesCustomInserter = 1, Defs = [NZCV] in {
 def ATOMIC_CMP_SWAP_I8
diff --git a/test/CodeGen/AArch64/atomic-ops.ll b/test/CodeGen/AArch64/atomic-ops.ll
index bcb1a6f614..f3c16171cc 100644
--- a/test/CodeGen/AArch64/atomic-ops.ll
+++ b/test/CodeGen/AArch64/atomic-ops.ll
@@ -18,8 +18,7 @@ define i8 @test_atomic_load_add_i8(i8 %offset) nounwind {
   ;  function there.
 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -39,8 +38,7 @@ define i16 @test_atomic_load_add_i16(i16 %offset) nounwind {
   ;  function there.
 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -60,8 +58,7 @@ define i32 @test_atomic_load_add_i32(i32 %offset) nounwind {
   ;  function there.
 ; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -81,8 +78,7 @@ define i64 @test_atomic_load_add_i64(i64 %offset) nounwind {
   ; function there.
 ; CHECK-NEXT: add [[NEW:x[0-9]+]], x[[OLD]], x0
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -102,8 +98,7 @@ define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
   ;  function there.
 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -123,8 +118,7 @@ define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {
   ;  function there.
 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -144,8 +138,7 @@ define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {
   ;  function there.
 ; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -165,8 +158,7 @@ define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind {
   ; function there.
 ; CHECK-NEXT: sub [[NEW:x[0-9]+]], x[[OLD]], x0
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -186,8 +178,7 @@ define i8 @test_atomic_load_and_i8(i8 %offset) nounwind {
   ;  function there.
 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -207,8 +198,7 @@ define i16 @test_atomic_load_and_i16(i16 %offset) nounwind {
   ;  function there.
 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -228,8 +218,7 @@ define i32 @test_atomic_load_and_i32(i32 %offset) nounwind {
   ;  function there.
 ; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -249,8 +238,7 @@ define i64 @test_atomic_load_and_i64(i64 %offset) nounwind {
   ; function there.
 ; CHECK-NEXT: and [[NEW:x[0-9]+]], x[[OLD]], x0
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -270,8 +258,7 @@ define i8 @test_atomic_load_or_i8(i8 %offset) nounwind {
   ;  function there.
 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -291,8 +278,7 @@ define i16 @test_atomic_load_or_i16(i16 %offset) nounwind {
   ;  function there.
 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -312,8 +298,7 @@ define i32 @test_atomic_load_or_i32(i32 %offset) nounwind {
   ;  function there.
 ; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -333,8 +318,7 @@ define i64 @test_atomic_load_or_i64(i64 %offset) nounwind {
   ; function there.
 ; CHECK-NEXT: orr [[NEW:x[0-9]+]], x[[OLD]], x0
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -354,8 +338,7 @@ define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {
   ;  function there.
 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -375,8 +358,7 @@ define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {
   ;  function there.
 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -396,8 +378,7 @@ define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {
   ;  function there.
 ; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -417,8 +398,7 @@ define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
   ; function there.
 ; CHECK-NEXT: eor [[NEW:x[0-9]+]], x[[OLD]], x0
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -437,8 +417,7 @@ define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
   ; w0 below is a reasonable guess but could change: it certainly comes into the
   ; function there.
 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -457,8 +436,7 @@ define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
   ; w0 below is a reasonable guess but could change: it certainly comes into the
   ; function there.
 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -477,8 +455,7 @@ define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {
   ; w0 below is a reasonable guess but could change: it certainly comes into the
   ;  function there.
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -497,8 +474,7 @@ define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind {
   ; x0 below is a reasonable guess but could change: it certainly comes into the
   ; function there.
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], x0, [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -520,8 +496,7 @@ define i8 @test_atomic_load_min_i8(i8 %offset) nounwind {
 ; CHECK-NEXT: cmp w0, w[[OLD]], sxtb
 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -542,8 +517,7 @@ define i16 @test_atomic_load_min_i16(i16 %offset) nounwind {
 ; CHECK-NEXT: cmp w0, w[[OLD]], sxth
 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -564,8 +538,7 @@ define i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
 ; CHECK-NEXT: cmp w0, w[[OLD]]
 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -586,8 +559,7 @@ define i64 @test_atomic_load_min_i64(i64 %offset) nounwind {
 ; CHECK-NEXT: cmp x0, x[[OLD]]
 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, gt
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -608,8 +580,7 @@ define i8 @test_atomic_load_max_i8(i8 %offset) nounwind {
 ; CHECK-NEXT: cmp w0, w[[OLD]], sxtb
 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lt
 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -630,8 +601,7 @@ define i16 @test_atomic_load_max_i16(i16 %offset) nounwind {
 ; CHECK-NEXT: cmp w0, w[[OLD]], sxth
 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lt
 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -652,8 +622,7 @@ define i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
 ; CHECK-NEXT: cmp w0, w[[OLD]]
 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lt
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -674,8 +643,7 @@ define i64 @test_atomic_load_max_i64(i64 %offset) nounwind {
 ; CHECK-NEXT: cmp x0, x[[OLD]]
 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, lt
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -696,8 +664,7 @@ define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {
 ; CHECK-NEXT: cmp w0, w[[OLD]], uxtb
 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -718,8 +685,7 @@ define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {
 ; CHECK-NEXT: cmp w0, w[[OLD]], uxth
 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -740,8 +706,7 @@ define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
 ; CHECK-NEXT: cmp w0, w[[OLD]]
 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -762,8 +727,7 @@ define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {
 ; CHECK-NEXT: cmp x0, x[[OLD]]
 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, hi
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -784,8 +748,7 @@ define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {
 ; CHECK-NEXT: cmp w0, w[[OLD]], uxtb
 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lo
 ; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -806,8 +769,7 @@ define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {
 ; CHECK-NEXT: cmp w0, w[[OLD]], uxth
 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lo
 ; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -828,8 +790,7 @@ define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
 ; CHECK-NEXT: cmp w0, w[[OLD]]
 ; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lo
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -850,8 +811,7 @@ define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
 ; CHECK-NEXT: cmp x0, x[[OLD]]
 ; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, lo
 ; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne .LBB{{[0-9]+}}_1
+; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -873,8 +833,7 @@ define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
   ; As above, w1 is a reasonable guess.
 ; CHECK: stxrb [[STATUS:w[0-9]+]], w1, [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne [[STARTAGAIN]]
+; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -896,8 +855,7 @@ define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
   ; As above, w1 is a reasonable guess.
 ; CHECK: stxrh [[STATUS:w[0-9]+]], w1, [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne [[STARTAGAIN]]
+; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -919,8 +877,7 @@ define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
   ; As above, w1 is a reasonable guess.
 ; CHECK: stxr [[STATUS:w[0-9]+]], w1, [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne [[STARTAGAIN]]
+; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]
@@ -942,8 +899,7 @@ define i64 @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
 ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
   ; As above, w1 is a reasonable guess.
 ; CHECK: stxr [[STATUS:w[0-9]+]], x1, [x[[ADDR]]]
-; CHECK-NEXT: cmp [[STATUS]], #0
-; CHECK-NEXT: b.ne [[STARTAGAIN]]
+; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
 ; CHECK: dmb ish
 
 ; CHECK: mov x0, x[[OLD]]